From 1d7f1587eae71df1243e5061fba7569fcebb6eca Mon Sep 17 00:00:00 2001 From: Aron Kerekes Date: Mon, 30 Mar 2026 23:53:52 +0200 Subject: [PATCH 01/11] mike Signed-off-by: Aron Kerekes --- .github/workflows/cicd.yml | 10 +++--- .github/workflows/reusable-docs.yml | 52 +++++++++++++---------------- mkdocs/mkdocs.yml | 4 +++ mkdocs/pyproject.toml | 15 +++++---- mkdocs/uv.lock | 39 +++++++++++++++++++++- 5 files changed, 79 insertions(+), 41 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 7ab3976..d23210d 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -30,7 +30,11 @@ jobs: - name: Resolve required vars id: vars run: | - echo "release_tag=${GITHUB_REF#refs/tags/}" >> "$GITHUB_OUTPUT" + if [[ "$GITHUB_REF" == refs/tags/* ]]; then + echo "release_tag=${GITHUB_REF#refs/tags/}" >> "$GITHUB_OUTPUT" + else + echo "release_tag=dev" >> "$GITHUB_OUTPUT" + fi docs: name: Docs @@ -38,9 +42,7 @@ jobs: - prepare uses: ./.github/workflows/reusable-docs.yml permissions: - contents: read - pages: write - id-token: write + contents: write with: deploy: ${{ startsWith(github.ref, 'refs/tags/') }} version: ${{ needs.prepare.outputs.release_tag }} diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index ad82386..f583f49 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -31,13 +31,11 @@ on: default: dev permissions: - contents: read - pages: write - id-token: write + contents: write jobs: - build: - name: Build artifacts + docs: + name: Build and deploy runs-on: ubuntu-latest steps: - name: Checkout code @@ -46,10 +44,6 @@ jobs: fetch-depth: 0 submodules: 'recursive' - - name: Setup Pages - id: pages - uses: actions/configure-pages@v5 - - name: Setup Golang uses: actions/setup-go@v5 with: @@ -76,24 +70,24 @@ jobs: run: | task build - - name: Upload artifact - uses: actions/upload-pages-artifact@v4 - with: - name: docs-website - path: ./.build/site + - name: Configure Git for mike + if: ${{ inputs.deploy == true || inputs.deploy == 'true' }} + run: | + git config user.name github-actions[bot] + git config user.email github-actions[bot]@users.noreply.github.com - deploy: - name: Deploy artifacts - if: ${{ inputs.deploy == true || inputs.deploy == 'true' }} - needs: - - build - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 - with: - artifact_name: docs-website + - name: Fetch gh-pages + if: ${{ inputs.deploy == true || inputs.deploy == 'true' }} + continue-on-error: true + run: git fetch origin gh-pages --depth=1 + + - name: Deploy versioned docs (mike) + if: ${{ inputs.deploy == true || inputs.deploy == 'true' }} + shell: bash + working-directory: mkdocs + env: + VERSION: ${{ inputs.version }} + run: | + MIKE_VERSION="${VERSION#v}" + uv run mike deploy --push --update-aliases "$MIKE_VERSION" latest + uv run mike set-default --push latest diff --git a/mkdocs/mkdocs.yml b/mkdocs/mkdocs.yml index 46e5ad8..2b02a93 100644 --- a/mkdocs/mkdocs.yml +++ b/mkdocs/mkdocs.yml @@ -8,6 +8,8 @@ repo_url: https://github.com/agntcy/docs edit_uri: edit/main/docs/ extra: + version: + provider: mike copyright: > © Copyright AGNTCY Contributors. Change cookie settings @@ -50,6 +52,8 @@ hooks: plugins: search: + mike: + alias_type: redirect redirects: redirect_maps: 'how-to-guides/identity-quickstart/index.md': 'identity/identity-quickstart.md' diff --git a/mkdocs/pyproject.toml b/mkdocs/pyproject.toml index c00e63e..b72af51 100644 --- a/mkdocs/pyproject.toml +++ b/mkdocs/pyproject.toml @@ -18,14 +18,14 @@ dependencies = [ "mkdocstrings-python==1.16.10", "griffe-pydantic==1.1.4", "griffe==1.7.2", - "agntcy-acp>=1.5.2", # 1.5.x allows langgraph 0.4+ so we can use langgraph-checkpoint>=3.0 + "agntcy-acp>=1.5.2", # 1.5.x allows langgraph 0.4+ so we can use langgraph-checkpoint>=3.0 "agntcy-iomapper==0.2.2", - "langgraph-checkpoint>=3.0.0", # JsonPlusSerializer RCE in "json" mode - "llama-index>=0.12.38", # JSONReader DoS (uncontrolled recursion) fixed in 0.12.38 - "llama-index-core>=0.12.38,<0.13", # pin to 0.12.x so resolver does not pick 0.13.0 on Python 3.12.4+ - "nltk>=3.9.3", # CVE-2025-14009: zip slip / RCE in downloader - "orjson>=3.11.6", # CVE-2025-67221: recursion limit for deeply nested JSON - "pillow>=12.1.1", # CVE-2025-48379, CVE-2026-25990 + "langgraph-checkpoint>=3.0.0", # JsonPlusSerializer RCE in "json" mode + "llama-index>=0.12.38", # JSONReader DoS (uncontrolled recursion) fixed in 0.12.38 + "llama-index-core>=0.12.38,<0.13", # pin to 0.12.x so resolver does not pick 0.13.0 on Python 3.12.4+ + "nltk>=3.9.3", # CVE-2025-14009: zip slip / RCE in downloader + "orjson>=3.11.6", # CVE-2025-67221: recursion limit for deeply nested JSON + "pillow>=12.1.1", # CVE-2025-48379, CVE-2026-25990 "beautifulsoup4>=4.13.3", "mkdocs-swagger-ui-tag==0.7.1", "mkdocs-include-markdown-plugin>=7.2.0", @@ -35,4 +35,5 @@ dependencies = [ "pymarkdownlnt>=0.9.0", "mkdocs-htmlproofer-plugin>=1.0.0", "certifi>=2025.1.31", + "mike>=2.1.4", ] diff --git a/mkdocs/uv.lock b/mkdocs/uv.lock index 80d1ae4..b4a035c 100644 --- a/mkdocs/uv.lock +++ b/mkdocs/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.12.4' and python_full_version < '4'", @@ -46,6 +46,7 @@ dependencies = [ { name = "llama-index" }, { name = "llama-index-core" }, { name = "markdown" }, + { name = "mike" }, { name = "mkdocs" }, { name = "mkdocs-autorefs" }, { name = "mkdocs-awesome-pages-plugin" }, @@ -79,6 +80,7 @@ requires-dist = [ { name = "llama-index", specifier = ">=0.12.38" }, { name = "llama-index-core", specifier = ">=0.12.38,<0.13" }, { name = "markdown", specifier = ">=3.2" }, + { name = "mike", specifier = ">=2.1.4" }, { name = "mkdocs", specifier = ">=1.3.0" }, { name = "mkdocs-autorefs", specifier = "==1.4.1" }, { name = "mkdocs-awesome-pages-plugin", specifier = "==2.10.1" }, @@ -1658,6 +1660,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354, upload-time = "2021-02-05T18:55:29.583Z" }, ] +[[package]] +name = "mike" +version = "2.1.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "mkdocs" }, + { name = "pyparsing" }, + { name = "pyyaml" }, + { name = "pyyaml-env-tag" }, + { name = "verspec" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ec/09/de1cab0018eb5f1fbd9dcc26b6e61f9453c5ec2eb790949d6ed75e1ffe55/mike-2.1.4.tar.gz", hash = "sha256:75d549420b134603805a65fc67f7dcd9fcd0ad1454fb2c893d9e844cba1aa6e4", size = 38190, upload-time = "2026-03-08T02:46:29.187Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/f7/10f5e101db25741b91e4f4792c5d97b4fa834ead5cf509ae91097d939424/mike-2.1.4-py3-none-any.whl", hash = "sha256:39933e992e155dd70f2297e749a0ed78d8fd7942bc33a3666195d177758a280e", size = 33820, upload-time = "2026-03-08T02:46:28.149Z" }, +] + [[package]] name = "mkdocs" version = "1.6.1" @@ -2820,6 +2839,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/06/43084e6cbd4b3bc0e80f6be743b2e79fbc6eed8de9ad8c629939fa55d972/pymdown_extensions-10.16.1-py3-none-any.whl", hash = "sha256:d6ba157a6c03146a7fb122b2b9a121300056384eafeec9c9f9e584adfdb2a32d", size = 266178, upload-time = "2025-07-28T16:19:31.401Z" }, ] +[[package]] +name = "pyparsing" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, +] + [[package]] name = "pypdf" version = "5.4.0" @@ -3501,6 +3529,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ed/d0/5bf7cbf1ac138c92b9ac21066d18faf4d7e7f651047b700eb192ca4b9fdb/uuid_utils-0.14.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:258186964039a8e36db10810c1ece879d229b01331e09e9030bc5dcabe231bd2", size = 364700, upload-time = "2026-02-20T22:50:21.732Z" }, ] +[[package]] +name = "verspec" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/44/8126f9f0c44319b2efc65feaad589cadef4d77ece200ae3c9133d58464d0/verspec-0.1.0.tar.gz", hash = "sha256:c4504ca697b2056cdb4bfa7121461f5a0e81809255b41c03dda4ba823637c01e", size = 27123, upload-time = "2020-11-30T02:24:09.646Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/ce/3b6fee91c85626eaf769d617f1be9d2e15c1cca027bbdeb2e0d751469355/verspec-0.1.0-py3-none-any.whl", hash = "sha256:741877d5633cc9464c45a469ae2a31e801e6dbbaa85b9675d481cda100f11c31", size = 19640, upload-time = "2020-11-30T02:24:08.387Z" }, +] + [[package]] name = "watchdog" version = "6.0.0" From 19f2e5bd8d3beb807de3f222ef90a88da5b71acc Mon Sep 17 00:00:00 2001 From: Aron Kerekes Date: Tue, 31 Mar 2026 00:08:58 +0200 Subject: [PATCH 02/11] variables Signed-off-by: Aron Kerekes --- docs/stylesheets/custom.css | 25 +++++++++++++++++ mkdocs/main.py | 23 ++++++++++++++++ mkdocs/mkdocs.yml | 12 +++++++++ mkdocs/pyproject.toml | 1 + mkdocs/uv.lock | 53 +++++++++++++++++++++++++++++++++++++ 5 files changed, 114 insertions(+) create mode 100644 mkdocs/main.py diff --git a/docs/stylesheets/custom.css b/docs/stylesheets/custom.css index 063818c..372932a 100644 --- a/docs/stylesheets/custom.css +++ b/docs/stylesheets/custom.css @@ -151,4 +151,29 @@ html [data-md-color-primary=black] .md-nav--primary .md-nav__title[for=__drawer] [data-md-color-scheme="default"] code { background-color: rgb(246, 248, 250) !important; +} + +/* mkdocs-macros: {{ var_tag('NAME') }} */ +.doc-var-tag { + display: inline-flex; + align-items: center; + margin: 0 0.15em; + vertical-align: baseline; + border-radius: 0.25rem; + padding: 0.05em 0.35em; + font-size: 0.92em; + line-height: 1.35; + border: 1px solid var(--md-default-fg-color--lighter); + background-color: var(--md-default-bg-color--lightest); +} + +.doc-var-tag code { + background-color: transparent !important; + padding: 0 !important; + font-size: inherit !important; +} + +[data-md-color-scheme="slate"] .doc-var-tag { + border-color: var(--md-default-fg-color--lightest); + background-color: var(--md-code-bg-color); } \ No newline at end of file diff --git a/mkdocs/main.py b/mkdocs/main.py new file mode 100644 index 0000000..bf7c6c6 --- /dev/null +++ b/mkdocs/main.py @@ -0,0 +1,23 @@ +"""MkDocs macros: shared variables (from mkdocs.yml `extra`) and reusable helpers.""" + +from __future__ import annotations + +import html +import os + + +def define_env(env): + """Register variables and macros for mkdocs-macros-plugin.""" + + env.variables["docs_build_version"] = os.environ.get("VERSION", "dev") + + @env.macro + def var_tag(label: str, hint: str = "") -> str: + """Inline tag for env vars, flags, or keys. Usage: {{ var_tag('MY_ENV') }}.""" + safe_label = html.escape(str(label), quote=True) + safe_hint = html.escape(str(hint), quote=True) if hint else "" + title = f' title="{safe_hint}"' if safe_hint else "" + return ( + f'' + f"{safe_label}" + ) diff --git a/mkdocs/mkdocs.yml b/mkdocs/mkdocs.yml index 2b02a93..65bab9f 100644 --- a/mkdocs/mkdocs.yml +++ b/mkdocs/mkdocs.yml @@ -10,6 +10,12 @@ edit_uri: edit/main/docs/ extra: version: provider: mike + # Shared Jinja variables for mkdocs-macros (use as {{ var.docs_url }}, etc.) + var: + docs_url: https://docs.agntcy.org/ + repo_url: https://github.com/agntcy/docs + repo_slug: agntcy/docs + org: agntcy copyright: > © Copyright AGNTCY Contributors. Change cookie settings @@ -75,6 +81,12 @@ plugins: filename: ".index" collapse_single_pages: true strict: false + macros: + # main.py in this directory defines env.variables and @env.macro helpers + module_name: main + # Avoid clashing with GitHub Actions (${{ }}) and JSX ({{ }}) in fenced examples + j2_variable_start_string: "[[[" + j2_variable_end_string: "]]]" mkdocstrings: handlers: python: diff --git a/mkdocs/pyproject.toml b/mkdocs/pyproject.toml index b72af51..02fc884 100644 --- a/mkdocs/pyproject.toml +++ b/mkdocs/pyproject.toml @@ -36,4 +36,5 @@ dependencies = [ "mkdocs-htmlproofer-plugin>=1.0.0", "certifi>=2025.1.31", "mike>=2.1.4", + "mkdocs-macros-plugin>=1.5.0", ] diff --git a/mkdocs/uv.lock b/mkdocs/uv.lock index b4a035c..9bac532 100644 --- a/mkdocs/uv.lock +++ b/mkdocs/uv.lock @@ -54,6 +54,7 @@ dependencies = [ { name = "mkdocs-git-revision-date-plugin" }, { name = "mkdocs-htmlproofer-plugin" }, { name = "mkdocs-include-markdown-plugin" }, + { name = "mkdocs-macros-plugin" }, { name = "mkdocs-material" }, { name = "mkdocs-material-extensions" }, { name = "mkdocs-redirects" }, @@ -88,6 +89,7 @@ requires-dist = [ { name = "mkdocs-git-revision-date-plugin", specifier = "==0.3.2" }, { name = "mkdocs-htmlproofer-plugin", specifier = ">=1.0.0" }, { name = "mkdocs-include-markdown-plugin", specifier = ">=7.2.0" }, + { name = "mkdocs-macros-plugin", specifier = ">=1.5.0" }, { name = "mkdocs-material", specifier = "==9.6.12" }, { name = "mkdocs-material-extensions", specifier = ">=1.0.3" }, { name = "mkdocs-redirects", specifier = ">=1.2.0" }, @@ -992,6 +994,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "hjson" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/e5/0b56d723a76ca67abadbf7fb71609fb0ea7e6926e94fcca6c65a85b36a0e/hjson-3.1.0.tar.gz", hash = "sha256:55af475a27cf83a7969c808399d7bccdec8fb836a07ddbd574587593b9cdcf75", size = 40541, upload-time = "2022-08-13T02:53:01.919Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/7f/13cd798d180af4bf4c0ceddeefba2b864a63c71645abc0308b768d67bb81/hjson-3.1.0-py3-none-any.whl", hash = "sha256:65713cdcf13214fb554eb8b4ef803419733f4f5e551047c9b711098ab7186b89", size = 54018, upload-time = "2022-08-13T02:52:59.899Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -1793,6 +1804,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/f9/783338d1d7fd548c7635728b67a0f8f96d9e6c265aa61c51356c03597767/mkdocs_include_markdown_plugin-7.2.0-py3-none-any.whl", hash = "sha256:d56cdaeb2d113fb66ed0fe4fb7af1da889926b0b9872032be24e19bbb09c9f5b", size = 29548, upload-time = "2025-09-28T21:50:49.373Z" }, ] +[[package]] +name = "mkdocs-macros-plugin" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hjson" }, + { name = "jinja2" }, + { name = "mkdocs" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "super-collections" }, + { name = "termcolor" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/92/15/e6a44839841ebc9c5872fa0e6fad1c3757424e4fe026093b68e9f386d136/mkdocs_macros_plugin-1.5.0.tar.gz", hash = "sha256:12aa45ce7ecb7a445c66b9f649f3dd05e9b92e8af6bc65e4acd91d26f878c01f", size = 37730, upload-time = "2025-11-13T08:08:55.545Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/62/9fffba5bb9ed3d31a932ad35038ba9483d59850256ee0fea7f1187173983/mkdocs_macros_plugin-1.5.0-py3-none-any.whl", hash = "sha256:c10fabd812bf50f9170609d0ed518e54f1f0e12c334ac29141723a83c881dd6f", size = 44626, upload-time = "2025-11-13T08:08:53.878Z" }, +] + [[package]] name = "mkdocs-material" version = "9.6.12" @@ -3316,6 +3348,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/cf/0fea4f4ba3fc2772ac2419278aa9f6964124d4302117d61bc055758e000c/striprtf-0.0.26-py3-none-any.whl", hash = "sha256:8c8f9d32083cdc2e8bfb149455aa1cc5a4e0a035893bedc75db8b73becb3a1bb", size = 6914, upload-time = "2023-07-20T14:30:35.338Z" }, ] +[[package]] +name = "super-collections" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hjson" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/de/a0c3d1244912c260638f0f925e190e493ccea37ecaea9bbad7c14413b803/super_collections-0.6.2.tar.gz", hash = "sha256:0c8d8abacd9fad2c7c1c715f036c29f5db213f8cac65f24d45ecba12b4da187a", size = 31315, upload-time = "2025-09-30T00:37:08.067Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/43/47c7cf84b3bd74a8631b02d47db356656bb8dff6f2e61a4c749963814d0d/super_collections-0.6.2-py3-none-any.whl", hash = "sha256:291b74d26299e9051d69ad9d89e61b07b6646f86a57a2f5ab3063d206eee9c56", size = 16173, upload-time = "2025-09-30T00:37:07.104Z" }, +] + [[package]] name = "tenacity" version = "9.1.2" @@ -3325,6 +3369,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" }, ] +[[package]] +name = "termcolor" +version = "3.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/79/cf31d7a93a8fdc6aa0fbb665be84426a8c5a557d9240b6239e9e11e35fc5/termcolor-3.3.0.tar.gz", hash = "sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5", size = 14434, upload-time = "2025-12-29T12:55:21.882Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl", hash = "sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5", size = 7734, upload-time = "2025-12-29T12:55:20.718Z" }, +] + [[package]] name = "tiktoken" version = "0.9.0" From 2aa027e1cba495eb0425122fb24f64c71740fe2c Mon Sep 17 00:00:00 2001 From: Aron Kerekes Date: Tue, 31 Mar 2026 00:09:17 +0200 Subject: [PATCH 03/11] more variables Signed-off-by: Aron Kerekes --- docs/contributing.md | 22 ++++++++++++++++++++++ mkdocs/README.md | 20 ++++++++++++++++++++ mkdocs/mkdocs.yml | 5 +++-- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/docs/contributing.md b/docs/contributing.md index 0b5e763..5912eae 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -159,3 +159,25 @@ Is displayed as: This is a note. For more information on formatting, see the [Material for MkDocs reference](https://squidfunk.github.io/mkdocs-material/reference/). + +## Variables + +Variable tags are used to display variables in the docs such as version numbers and component names. + +[mkdocs-macros-plugin](https://mkdocs-macros-plugin.readthedocs.io/) expands `[[[ ... ]]]` on the whole page source before Markdown runs, so substitutions work inside fenced code blocks as well as in prose: + +````markdown +```bash +curl "[[[ var.docs_url ]]]" +``` +```` + +To show the literal characters `[[[ var.org ]]]` in the docs (e.g. in a macro how-to), wrap that part in a Jinja raw block (block delimiters stay the default `{%` / `%}`): + +````markdown +{% raw %} +```text +[[[ var.org ]]] +``` +{% endraw %} +```` \ No newline at end of file diff --git a/mkdocs/README.md b/mkdocs/README.md index fc2808f..120902f 100644 --- a/mkdocs/README.md +++ b/mkdocs/README.md @@ -11,3 +11,23 @@ uv run mkdocs serve ```sh uv run mkdocs build ``` + +## Macros + +[mkdocs-macros-plugin](https://mkdocs-macros-plugin.readthedocs.io/) expands **`[[[ ... ]]]`** on the **whole page source before** Markdown runs, so substitutions work **inside fenced code blocks** as well as in prose: + +````markdown +```bash +curl "[[[ var.docs_url ]]]" +``` +```` + +To show the literal characters `[[[ var.org ]]]` in the docs (e.g. in a macro how-to), wrap that part in a Jinja raw block (block delimiters stay the default `{%` / `%}`): + +````markdown +{% raw %} +```text +[[[ var.org ]]] +``` +{% endraw %} +```` diff --git a/mkdocs/mkdocs.yml b/mkdocs/mkdocs.yml index 65bab9f..83503c8 100644 --- a/mkdocs/mkdocs.yml +++ b/mkdocs/mkdocs.yml @@ -10,12 +10,13 @@ edit_uri: edit/main/docs/ extra: version: provider: mike - # Shared Jinja variables for mkdocs-macros (use as {{ var.docs_url }}, etc.) + # Shared Jinja variables for mkdocs-macros (use as [[[ var.docs_url ]]], etc.) var: docs_url: https://docs.agntcy.org/ repo_url: https://github.com/agntcy/docs repo_slug: agntcy/docs - org: agntcy + org: AGNTCY + copyright: > © Copyright AGNTCY Contributors. Change cookie settings From 9989782feaa811d74b87b49ee7ff43221db0e1a1 Mon Sep 17 00:00:00 2001 From: Aron Kerekes Date: Tue, 31 Mar 2026 00:26:37 +0200 Subject: [PATCH 04/11] mike fixes Signed-off-by: Aron Kerekes --- Taskfile.yml | 21 +++++++++++++++++++++ docs/stylesheets/custom.css | 4 +++- mkdocs/README.md | 20 -------------------- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 226693c..7d342f1 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -40,6 +40,27 @@ tasks: cmds: - pushd mkdocs && uv run mkdocs serve + mike:deploy-local: + desc: Deploy current docs to local gh-pages as dev + latest (no git push) + deps: + - deps/patch + dir: mkdocs + cmds: + - git fetch origin gh-pages --depth=1 2>/dev/null || true + - uv run mike deploy dev latest --update-aliases + - uv run mike set-default latest + - uv run mike list + - echo "Local gh-pages updated (dev/latest). Run task mike:serve and open http://127.0.0.1:8000/" + + mike:serve: + desc: Serve versioned site from local gh-pages (run task mike:deploy-local first) + deps: + - deps/patch + dir: mkdocs + cmds: + # Bind to 127.0.0.1 so the URL is stable (avoids reverse-DNS localhost like 1.0.0.127.in-addr.arpa) + - uv run mike serve -a 127.0.0.1:8000 + ## ## Testing and Linting ## diff --git a/docs/stylesheets/custom.css b/docs/stylesheets/custom.css index 372932a..9f1cbfd 100644 --- a/docs/stylesheets/custom.css +++ b/docs/stylesheets/custom.css @@ -145,7 +145,9 @@ html [data-md-color-primary=black] .md-nav--primary .md-nav__title[for=__drawer] font-size: initial !important; } -.md-header__ellipsis> :first-child { +/* Hide the duplicate site name in the header; do NOT hide the whole first + .md-header__topic — Material appends the mike version selector there. */ +.md-header__ellipsis > .md-header__topic:first-child > .md-ellipsis { display: none !important; } diff --git a/mkdocs/README.md b/mkdocs/README.md index 120902f..fc2808f 100644 --- a/mkdocs/README.md +++ b/mkdocs/README.md @@ -11,23 +11,3 @@ uv run mkdocs serve ```sh uv run mkdocs build ``` - -## Macros - -[mkdocs-macros-plugin](https://mkdocs-macros-plugin.readthedocs.io/) expands **`[[[ ... ]]]`** on the **whole page source before** Markdown runs, so substitutions work **inside fenced code blocks** as well as in prose: - -````markdown -```bash -curl "[[[ var.docs_url ]]]" -``` -```` - -To show the literal characters `[[[ var.org ]]]` in the docs (e.g. in a macro how-to), wrap that part in a Jinja raw block (block delimiters stay the default `{%` / `%}`): - -````markdown -{% raw %} -```text -[[[ var.org ]]] -``` -{% endraw %} -```` From d6edda699c82f75790ea374db090d4a4694ed65b Mon Sep 17 00:00:00 2001 From: Aron Kerekes Date: Tue, 31 Mar 2026 00:36:17 +0200 Subject: [PATCH 05/11] mike Signed-off-by: Aron Kerekes --- .github/workflows/reusable-docs.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index f583f49..27b3d05 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -37,6 +37,9 @@ jobs: docs: name: Build and deploy runs-on: ubuntu-latest + env: + # Use Node 24 for JS-based actions (checkout, etc.); see GitHub deprecation of Node 20 on runners + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true steps: - name: Checkout code uses: actions/checkout@v4 @@ -44,11 +47,6 @@ jobs: fetch-depth: 0 submodules: 'recursive' - - name: Setup Golang - uses: actions/setup-go@v5 - with: - go-version: '1.23.1' - - name: Setup Taskfile shell: bash run: | From 2ae9de9762d9010d3da20bee0ab3908227efc191 Mon Sep 17 00:00:00 2001 From: Aron Kerekes Date: Tue, 31 Mar 2026 17:12:45 +0200 Subject: [PATCH 06/11] mike config Signed-off-by: Aron Kerekes --- .github/workflows/cicd.yml | 23 +--- .github/workflows/docs-release.yml | 165 ++++++++++++++++++++++++++++ .github/workflows/reusable-docs.yml | 49 ++------- Taskfile.yml | 31 +++++- mkdocs/mike_version.py | 46 ++++++++ mkdocs/mike_versions.ini | 11 ++ 6 files changed, 260 insertions(+), 65 deletions(-) create mode 100644 .github/workflows/docs-release.yml create mode 100644 mkdocs/mike_version.py create mode 100644 mkdocs/mike_versions.ini diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index d23210d..0656b54 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -21,37 +21,18 @@ concurrency: cancel-in-progress: true jobs: - prepare: - name: Prepare - outputs: - release_tag: ${{ steps.vars.outputs.release_tag }} - runs-on: ubuntu-latest - steps: - - name: Resolve required vars - id: vars - run: | - if [[ "$GITHUB_REF" == refs/tags/* ]]; then - echo "release_tag=${GITHUB_REF#refs/tags/}" >> "$GITHUB_OUTPUT" - else - echo "release_tag=dev" >> "$GITHUB_OUTPUT" - fi - docs: name: Docs - needs: - - prepare uses: ./.github/workflows/reusable-docs.yml permissions: - contents: write + contents: read with: - deploy: ${{ startsWith(github.ref, 'refs/tags/') }} - version: ${{ needs.prepare.outputs.release_tag }} + version: dev success: name: Success if: ${{ !cancelled() && !contains(needs.*.result, 'cancelled') && !contains(needs.*.result, 'failure') }} needs: - - prepare - docs runs-on: ubuntu-latest steps: diff --git a/.github/workflows/docs-release.yml b/.github/workflows/docs-release.yml new file mode 100644 index 0000000..0974b7d --- /dev/null +++ b/.github/workflows/docs-release.yml @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Cisco and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +# +# Ordered release pipeline (decoupled phases): +# 1. Build docs — validate MkDocs output +# 2. Mike — push versioned site to gh-pages (from mkdocs/mike_versions.ini [versions] release) +# 3. Git tag — create v* tag on the commit that was published +# 4. Finalize — placeholder / optional GitHub Release (GitHub Pages already serves gh-pages after step 2) + +name: Docs release + +on: + workflow_dispatch: + inputs: + create_github_release: + description: 'After tagging, create a GitHub Release for the same tag' + type: choice + options: + - 'no' + - 'yes' + default: 'no' + +permissions: + contents: write + +concurrency: + group: docs-release + cancel-in-progress: false + +jobs: + build: + name: '1. Build docs' + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + - name: Setup Taskfile + shell: bash + run: sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin + + - name: Setup UV + shell: bash + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Update PATH + shell: bash + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Build + shell: bash + run: | + export VERSION="$(cd mkdocs && uv run python mike_version.py release)" + task build + + mike: + name: '2. Mike (gh-pages)' + needs: build + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + - name: Setup Taskfile + shell: bash + run: sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin + + - name: Setup UV + shell: bash + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Update PATH + shell: bash + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Configure Git for mike + run: | + git config user.name github-actions[bot] + git config user.email github-actions[bot]@users.noreply.github.com + + - name: Fetch gh-pages + continue-on-error: true + run: git fetch origin gh-pages --depth=1 + + - name: Mike deploy + shell: bash + run: task mike:deploy + + git-tag: + name: '3. Git tag' + needs: mike + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + - name: Setup UV + shell: bash + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Update PATH + shell: bash + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Create and push tag + shell: bash + env: + TAG_SHA: ${{ github.sha }} + run: | + set -euo pipefail + VER="$(cd mkdocs && uv run python mike_version.py release)" + TAG="v${VER}" + if git ls-remote --tags origin "refs/tags/${TAG}" | grep -q .; then + echo "::error::Tag ${TAG} already exists on origin. Bump [versions] release in mkdocs/mike_versions.ini or delete the remote tag." + exit 1 + fi + git tag -a "${TAG}" -m "Documentation release ${TAG}" "${TAG_SHA}" + git push origin "${TAG}" + + finalize: + name: '4. Deploy / release' + needs: git-tag + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup UV + shell: bash + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Update PATH + shell: bash + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: GitHub Release + if: ${{ github.event.inputs.create_github_release == 'yes' }} + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + VER="$(cd mkdocs && uv run python mike_version.py release)" + TAG="v${VER}" + gh release create "${TAG}" --title "Documentation ${TAG}" --notes "Published from workflow [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}). Site is served from the gh-pages branch." + + - name: Pages note + if: ${{ github.event.inputs.create_github_release != 'yes' }} + run: | + echo "::notice::GitHub Pages (if configured from gh-pages) is already serving the mike version from step 2. Set **create_github_release** to **yes** to open a GitHub Release on the new tag." diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 27b3d05..091ec3f 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -1,41 +1,34 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 Cisco and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 +# +# Build-only documentation check. Versioned publish (mike), tagging, and releases +# use .github/workflows/docs-release.yml name: Documentation on: workflow_call: inputs: - deploy: - description: 'Deploy documentation artifacts' - required: true - type: boolean - default: false version: - description: 'Version to use for documentation artifacts' + description: 'MkDocs macros VERSION (docs_build_version)' required: true type: string default: dev workflow_dispatch: inputs: - deploy: - description: 'Deploy documentation artifacts' - required: true - type: boolean - default: false version: - description: 'Version to use for documentation artifacts' - required: true + description: 'MkDocs macros VERSION' + required: false type: string default: dev permissions: - contents: write + contents: read jobs: docs: - name: Build and deploy + name: Build docs runs-on: ubuntu-latest env: # Use Node 24 for JS-based actions (checkout, etc.); see GitHub deprecation of Node 20 on runners @@ -63,29 +56,7 @@ jobs: - name: Build docs shell: bash - env: - VERSION: ${{ inputs.version }} run: | + V="${{ inputs.version }}" + export VERSION="${V:-dev}" task build - - - name: Configure Git for mike - if: ${{ inputs.deploy == true || inputs.deploy == 'true' }} - run: | - git config user.name github-actions[bot] - git config user.email github-actions[bot]@users.noreply.github.com - - - name: Fetch gh-pages - if: ${{ inputs.deploy == true || inputs.deploy == 'true' }} - continue-on-error: true - run: git fetch origin gh-pages --depth=1 - - - name: Deploy versioned docs (mike) - if: ${{ inputs.deploy == true || inputs.deploy == 'true' }} - shell: bash - working-directory: mkdocs - env: - VERSION: ${{ inputs.version }} - run: | - MIKE_VERSION="${VERSION#v}" - uv run mike deploy --push --update-aliases "$MIKE_VERSION" latest - uv run mike set-default --push latest diff --git a/Taskfile.yml b/Taskfile.yml index 7d342f1..e06f9a6 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -41,16 +41,37 @@ tasks: - pushd mkdocs && uv run mkdocs serve mike:deploy-local: - desc: Deploy current docs to local gh-pages as dev + latest (no git push) + desc: >- + Deploy current docs to local gh-pages with version from mkdocs/mike_versions.ini [versions] local (+ latest alias); no git push deps: - deps/patch dir: mkdocs cmds: - git fetch origin gh-pages --depth=1 2>/dev/null || true - - uv run mike deploy dev latest --update-aliases - - uv run mike set-default latest - - uv run mike list - - echo "Local gh-pages updated (dev/latest). Run task mike:serve and open http://127.0.0.1:8000/" + - | + set -euo pipefail + MIKE_LOCAL="$(uv run python mike_version.py local)" + uv run mike deploy "$MIKE_LOCAL" latest --update-aliases + uv run mike set-default latest + uv run mike list + echo "Local gh-pages updated ($MIKE_LOCAL + latest). Run task mike:serve and open http://127.0.0.1:8000/" + + mike:deploy: + desc: >- + Push versioned docs to origin/gh-pages (mike). Not part of task build. + Version: export VERSION=… or set [versions] release in mkdocs/mike_versions.ini. + Requires git user.name/email. Run task build first to validate the site. + deps: + - deps/patch + dir: mkdocs + cmds: + - git fetch origin gh-pages --depth=1 2>/dev/null || true + - | + set -euo pipefail + MIKE_VERSION="$(uv run python mike_version.py release)" + uv run mike deploy --push --update-aliases "$MIKE_VERSION" latest + uv run mike set-default --push latest + uv run mike list mike:serve: desc: Serve versioned site from local gh-pages (run task mike:deploy-local first) diff --git a/mkdocs/mike_version.py b/mkdocs/mike_version.py new file mode 100644 index 0000000..c9f70ee --- /dev/null +++ b/mkdocs/mike_version.py @@ -0,0 +1,46 @@ +"""Print mike version from mike_versions.ini; optional VERSION env overrides release.""" + +from __future__ import annotations + +import argparse +import configparser +import os +from pathlib import Path + + +def strip_leading_v(value: str) -> str: + value = value.strip() + return value[1:] if value.startswith("v") else value + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("local", "release")) + args = parser.parse_args() + + ini = Path(__file__).resolve().parent / "mike_versions.ini" + cfg = configparser.ConfigParser() + if not cfg.read(ini, encoding="utf-8"): + raise SystemExit(f"could not read {ini}") + + if args.mode == "local": + raw = cfg.get("versions", "local", fallback="dev") + out = strip_leading_v(raw) or "dev" + print(out) + return + + env = os.environ.get("VERSION", "").strip() + if env: + print(strip_leading_v(env)) + return + + raw = cfg.get("versions", "release", fallback="").strip() + if not raw: + raise SystemExit( + "Set [versions] release in mkdocs/mike_versions.ini or export VERSION=..." + ) + print(strip_leading_v(raw)) + + +if __name__ == "__main__": + main() diff --git a/mkdocs/mike_versions.ini b/mkdocs/mike_versions.ini new file mode 100644 index 0000000..c2812c0 --- /dev/null +++ b/mkdocs/mike_versions.ini @@ -0,0 +1,11 @@ +# Mike version labels (edit by hand). +# +# local — task mike:deploy-local (no git push) +# release — task mike:deploy; GitHub: workflow "Docs release" (build → mike → tag → finalize). +# Bump release before running that workflow; tags are created after mike in CI. +# +# Optional: VERSION=v1.2.0 task mike:deploy overrides release for that run only. + +[versions] +local = dev +release = From 854373dc3435b8363a9bed1d917433f41f2ce3f9 Mon Sep 17 00:00:00 2001 From: Aron Kerekes Date: Wed, 1 Apr 2026 17:09:37 +0200 Subject: [PATCH 07/11] shared config template Signed-off-by: Aron Kerekes --- .shared-config/README.md | 131 ++++++++++ .shared-config/Taskfile.yml | 166 +++++++++++++ .shared-config/assets/favicon.ico | Bin 0 -> 2249 bytes .shared-config/assets/img/logo-dark.svg | 5 + .shared-config/assets/img/logo-light.svg | 3 + .shared-config/assets/logo.svg | 6 + .shared-config/codespellrc | 6 + .shared-config/github/workflows/cicd.yml | 44 ++++ .../github/workflows/docs-release.yml | 229 ++++++++++++++++++ .shared-config/github/workflows/links.yml | 71 ++++++ .shared-config/github/workflows/pr-links.yml | 135 +++++++++++ .../github/workflows/reusable-docs.yml | 75 ++++++ .shared-config/install.sh | 32 +++ .shared-config/lychee.toml | 25 ++ .shared-config/mkdocs/hooks.py | 13 + .shared-config/mkdocs/main.py.example | 23 ++ .shared-config/mkdocs/mike_version.py | 46 ++++ .shared-config/mkdocs/mike_versions.ini | 8 + .shared-config/mkdocs/mkdocs.template.yml | 152 ++++++++++++ .../mkdocs/overrides/partials/copyright.html | 3 + .../mkdocs/overrides/partials/logo.html | 2 + .shared-config/pymarkdown.yaml | 28 +++ .../requirements-agntcy-docs-lint.txt | 2 + .../requirements-agntcy-docs-theme.txt | 18 ++ .shared-config/stylesheets/custom.css | 181 ++++++++++++++ 25 files changed, 1404 insertions(+) create mode 100644 .shared-config/README.md create mode 100644 .shared-config/Taskfile.yml create mode 100644 .shared-config/assets/favicon.ico create mode 100644 .shared-config/assets/img/logo-dark.svg create mode 100644 .shared-config/assets/img/logo-light.svg create mode 100644 .shared-config/assets/logo.svg create mode 100644 .shared-config/codespellrc create mode 100644 .shared-config/github/workflows/cicd.yml create mode 100644 .shared-config/github/workflows/docs-release.yml create mode 100644 .shared-config/github/workflows/links.yml create mode 100644 .shared-config/github/workflows/pr-links.yml create mode 100644 .shared-config/github/workflows/reusable-docs.yml create mode 100755 .shared-config/install.sh create mode 100644 .shared-config/lychee.toml create mode 100644 .shared-config/mkdocs/hooks.py create mode 100644 .shared-config/mkdocs/main.py.example create mode 100644 .shared-config/mkdocs/mike_version.py create mode 100644 .shared-config/mkdocs/mike_versions.ini create mode 100644 .shared-config/mkdocs/mkdocs.template.yml create mode 100644 .shared-config/mkdocs/overrides/partials/copyright.html create mode 100644 .shared-config/mkdocs/overrides/partials/logo.html create mode 100644 .shared-config/pymarkdown.yaml create mode 100644 .shared-config/requirements-agntcy-docs-lint.txt create mode 100644 .shared-config/requirements-agntcy-docs-theme.txt create mode 100644 .shared-config/stylesheets/custom.css diff --git a/.shared-config/README.md b/.shared-config/README.md new file mode 100644 index 0000000..d0ece88 --- /dev/null +++ b/.shared-config/README.md @@ -0,0 +1,131 @@ +# AGNTCY shared docs theme + +Drop this `.shared-config` directory at the root of a repository to reuse the AGNTCY MkDocs Material look (CSS, logos, favicon, header logo partials). + +## Layout + +| Path | Purpose | +|------|---------| +| `stylesheets/custom.css` | Theme variables and layout tweaks (source) | +| `assets/` | `favicon.ico`, `img/logo-light.svg`, `img/logo-dark.svg`, optional `logo.svg` | +| `mkdocs/overrides/` | Material `partials` (light/dark logo swap, footer copyright block) | +| `mkdocs/mkdocs.template.yml` | Full starter `mkdocs.yml` (theme, mike, macros, redirects, nav, API docs, Swagger, includes, analytics) | +| `mkdocs/hooks.py` | SSL via `certifi` for `include-markdown` (referenced from the template; keep path in sync) | +| `mkdocs/mike_version.py` | Resolves version label from `mike_versions.ini` (for Task / CI) | +| `mkdocs/mike_versions.ini` | `[versions]` `local` / `release` labels for mike | +| `mkdocs/main.py.example` | Copy to `mkdocs/main.py` for `var_tag()` and `docs_build_version` | +| `requirements-agntcy-docs-theme.txt` | All template plugins (Material, mike, macros, redirects, awesome-pages, mkdocstrings, swagger UI, `include-markdown`, `certifi`, …) | +| `requirements-agntcy-docs-lint.txt` | `codespell` and `pymarkdown` for lint tasks | +| `Taskfile.yml` | Local build, serve (`run`), and lint tasks ([Task](https://taskfile.dev/) ≥ 3.35) | +| `codespellrc` | Spelling check config (used by `lint:spelling`) | +| `pymarkdown.yaml` | Markdown style config (used by `lint:markdown`) | +| `lychee.toml` | Link checker config (used by `lint:links`) | +| `install.sh` | Copies CSS and assets into `docs/`; optional `--with-mike-macros` installs mike + `main.py` | +| `github/workflows/` | CI, docs release (mike + tag), links, PR link checks — see below | + +## GitHub Actions + +Copy the YAML files from `github/workflows/` into `.github/workflows/` at the repository root (keep filenames). They assume `.shared-config/` exists beside `docs/` and `mkdocs/`. + +| Workflow | Role | +|----------|------| +| `reusable-docs.yml` | Build docs (`workflow_call` + `workflow_dispatch`); used by `cicd.yml`. | +| `cicd.yml` | On PR/push (paths: `docs/**`, `mkdocs/**`, `.shared-config/**`) and on `v*.*.*` tags, runs the reusable build. | +| `docs-release.yml` | Manual Docs release: build → mike deploy to gh-pages → git tag `v{version}` → optional GitHub Release. Set `[versions] release` in `mkdocs/mike_versions.ini` first. | +| `links.yml` | Scheduled / manual lychee on `.build/site`, config `.shared-config/lychee.toml`. | +| `pr-links.yml` | PRs to main/master: lychee on HTML for changed `docs/**/*.md` only. Add a `--remap` in the workflow if you need production URLs rewritten to the local build (see comment in the file). | + +Each workflow runs `./.shared-config/install.sh` (and `--with-mike-macros`) before `task -t .shared-config/Taskfile.yml build`, then `uv sync` in `mkdocs/` when `pyproject.toml` or `uv.lock` exists, otherwise `python3 -m pip install -r .shared-config/requirements-agntcy-docs-theme.txt`. + +Enable GitHub Pages from the gh-pages branch if you use mike. Grant Actions appropriate permissions (the release workflow uses `contents: write`). + +## What the template enables + +Aligned with `agntcy/docs` `mkdocs.yml`: + +- `hooks` — `../.shared-config/mkdocs/hooks.py` (paths are relative to the directory that contains `mkdocs/mkdocs.yml`); uses `certifi` for HTTPS when using `include-markdown`. +- `extra.version` / `mike` — versioned docs and Material’s version menu. +- `extra.var` + `macros` — `[[[ var.docs_url ]]]`, etc., with `[[[` / `]]]` delimiters; `main.py` from `main.py.example` (or `./install.sh --with-mike-macros`). +- `redirects` — same `redirect_maps` as the main docs site (trim or replace for a new project). +- `awesome-pages` — `.index` navigation files. +- `mkdocstrings` — Python API docs (install the packages you document in the same environment; `griffe-pydantic` is configured). +- `swagger-ui-tag` and `include-markdown` — OpenAPI embeds and Markdown includes. +- `extra.analytics` / `extra.consent` — Google Analytics + cookie consent; replace `G-XXXXXXXXXX` with your Measurement ID (or remove those blocks). + +Copy `mike_version.py`, `mike_versions.ini`, and `main.py` into `mkdocs/` when using mike/macros: + +```bash +./.shared-config/install.sh --with-mike-macros +``` + +Use `task -t .shared-config/Taskfile.yml mike:deploy-local`, `mike:deploy`, and `mike:serve` (requires mike via `uv run` or on `PATH`). + +**Important:** Copy `mkdocs.template.yml` into `mkdocs/mkdocs.yml` (do not point `-f` at the template inside `.shared-config/`, or `docs_dir`, `hooks`, and `theme.custom_dir` paths will resolve incorrectly). + +## Build and lint with Task + +From the repository root (expects `docs/` and `mkdocs/mkdocs.yml`): + +```bash +task -t .shared-config/Taskfile.yml build # → .build/site +task -t .shared-config/Taskfile.yml run # mkdocs serve +task -t .shared-config/Taskfile.yml lint # spelling + markdown + links +task -t .shared-config/Taskfile.yml lint:fix # auto-fix spelling + markdown +task -t .shared-config/Taskfile.yml mike:deploy-local # local gh-pages + mike (after install.sh --with-mike-macros) +task -t .shared-config/Taskfile.yml mike:serve +``` + +Or include the file from your main `Taskfile.yml`: + +```yaml +includes: + docs: .shared-config/Taskfile.yml +``` + +Then run namespaced tasks such as `task docs:build`. + +**Tools:** `lint:links` needs [lychee](https://github.com/lycheeverse/lychee) on your `PATH`. Spelling and Markdown lint use either `uv run` (if `mkdocs/uv.lock` or `mkdocs/pyproject.toml` exists) or global `codespell` / `pymarkdown` after: + +```bash +pip install -r .shared-config/requirements-agntcy-docs-lint.txt +``` + +Tune `codespellrc`, `pymarkdown.yaml`, and `lychee.toml` in `.shared-config/` for your project (for example, extend `ignore-words-list` or `exclude` URL patterns). + +## Why `install.sh`? + +MkDocs loads `extra_css` and theme static paths relative to `docs_dir`. Files under `.shared-config/` are not inside `docs/` unless you copy or symlink them. Running `install.sh` places: + +- `docs/stylesheets/agntcy-docs.css` +- `docs/assets/agntcy/favicon.ico` +- `docs/assets/agntcy/img/logo-*.svg` + +Re-run after updating `.shared-config` from upstream. + +## Quick start + +1. Add or create a `docs/` tree with at least `docs/index.md`. +2. From the repository root: + + ```bash + chmod +x .shared-config/install.sh + ./.shared-config/install.sh + ./.shared-config/install.sh --with-mike-macros # optional: mike + macros files in mkdocs/ + ``` + +3. Copy `.shared-config/mkdocs/mkdocs.template.yml` to `mkdocs/mkdocs.yml` and set `site_name`, `site_url`, `repo_*`, `edit_uri`, `extra.var`, `extra.analytics.property`, and edit `plugins.redirects` as needed. Remove plugins you do not use (for example `mkdocstrings` if you have no API pages). +4. Install deps and serve: + + ```bash + cd mkdocs + pip install -r ../.shared-config/requirements-agntcy-docs-theme.txt + mkdocs serve + ``` + +If `mkdocs.yml` lives somewhere else, adjust `docs_dir`, `theme.custom_dir`, and paths in the template so `custom_dir` still points at `.shared-config/mkdocs/overrides` relative to that file. + +## Merging into an existing site + +- Set `theme.custom_dir` to `../.shared-config/mkdocs/overrides` (from `mkdocs/mkdocs.yml`). +- Add `stylesheets/agntcy-docs.css` to `extra_css` (after running `install.sh`). +- Set `theme.favicon`, `theme.logo_light`, and `theme.logo_dark` to the `assets/agntcy/...` paths from the template, or merge the `theme.palette` / `theme.features` blocks as needed. diff --git a/.shared-config/Taskfile.yml b/.shared-config/Taskfile.yml new file mode 100644 index 0000000..5365cf2 --- /dev/null +++ b/.shared-config/Taskfile.yml @@ -0,0 +1,166 @@ +# Copyright AGNTCY Contributors (https://github.com/agntcy) +# SPDX-License-Identifier: CC-BY-4.0 +# +# Portable docs tasks. Requires Task >= 3.35 (TASKFILE_DIR). +# From repo root: task -t .shared-config/Taskfile.yml build +# Or include in your Taskfile: includes: { agntcy-docs: .shared-config/Taskfile.yml } + +version: "3" + +interval: "500ms" + +vars: + SHARED: "{{.TASKFILE_DIR}}" + REPO: + sh: cd "{{.SHARED}}/.." && pwd + DOCS: "{{.REPO}}/docs" + MKDOCS: "{{.REPO}}/mkdocs" + BUILD_SITE: "{{.REPO}}/.build/site" + SPELL_CONFIG: "{{.SHARED}}/codespellrc" + MD_CONFIG: "{{.SHARED}}/pymarkdown.yaml" + LYCHEE_CONFIG: "{{.SHARED}}/lychee.toml" + +tasks: + default: + desc: List tasks + cmds: + - task --list + + build: + desc: Build documentation site to .build/site + cmds: + - mkdir -p "{{.BUILD_SITE}}" + - | + if [ -f "{{.MKDOCS}}/uv.lock" ] || [ -f "{{.MKDOCS}}/pyproject.toml" ]; then + (cd "{{.MKDOCS}}" && uv run mkdocs build --site-dir "{{.BUILD_SITE}}") + else + (cd "{{.MKDOCS}}" && mkdocs build --site-dir "{{.BUILD_SITE}}") + fi + - 'echo "Docs built: file://{{.BUILD_SITE}}/index.html"' + + run: + desc: Run mkdocs serve (live reload) + cmds: + - | + if [ -f "{{.MKDOCS}}/uv.lock" ] || [ -f "{{.MKDOCS}}/pyproject.toml" ]; then + (cd "{{.MKDOCS}}" && uv run mkdocs serve) + else + (cd "{{.MKDOCS}}" && mkdocs serve) + fi + + test: + desc: Run all documentation lint checks + cmds: + - task: lint + + lint: + desc: Run spelling, markdown, and link checks + cmds: + - task: lint:spelling + - task: lint:markdown + - task: lint:links + + lint:spelling: + desc: Check spelling under docs/ + cmds: + - | + if [ -f "{{.MKDOCS}}/uv.lock" ] || [ -f "{{.MKDOCS}}/pyproject.toml" ]; then + (cd "{{.MKDOCS}}" && uv run codespell --config "{{.SPELL_CONFIG}}" "{{.DOCS}}") + else + codespell --config "{{.SPELL_CONFIG}}" "{{.DOCS}}" + fi + + lint:markdown: + desc: Check Markdown style under docs/ + cmds: + - | + if [ -f "{{.MKDOCS}}/uv.lock" ] || [ -f "{{.MKDOCS}}/pyproject.toml" ]; then + (cd "{{.MKDOCS}}" && uv run pymarkdown --config "{{.MD_CONFIG}}" scan "{{.DOCS}}") + else + pymarkdown --config "{{.MD_CONFIG}}" scan "{{.DOCS}}" + fi + + lint:links: + desc: Check links in markdown under docs/ (requires lychee on PATH) + cmds: + - lychee --config "{{.LYCHEE_CONFIG}}" "{{.DOCS}}" + + lint:fix: + desc: Auto-fix spelling and markdown where supported + cmds: + - task: lint:fix:spelling + - task: lint:fix:markdown + + lint:fix:spelling: + desc: Apply codespell write-fixes to docs/ + cmds: + - | + if [ -f "{{.MKDOCS}}/uv.lock" ] || [ -f "{{.MKDOCS}}/pyproject.toml" ]; then + (cd "{{.MKDOCS}}" && uv run codespell --config "{{.SPELL_CONFIG}}" "{{.DOCS}}" --write-changes) + else + codespell --config "{{.SPELL_CONFIG}}" "{{.DOCS}}" --write-changes + fi + + lint:fix:markdown: + desc: Apply pymarkdown fixes under docs/ + cmds: + - | + if [ -f "{{.MKDOCS}}/uv.lock" ] || [ -f "{{.MKDOCS}}/pyproject.toml" ]; then + (cd "{{.MKDOCS}}" && uv run pymarkdown --config "{{.MD_CONFIG}}" fix -r "{{.DOCS}}") + else + pymarkdown --config "{{.MD_CONFIG}}" fix -r "{{.DOCS}}" + fi + + mike:deploy-local: + desc: >- + Deploy current docs to local gh-pages with version from mkdocs/mike_versions.ini [versions] local (+ latest alias); no git push + dir: "{{.MKDOCS}}" + cmds: + - 'test -f mike_version.py || { echo "Run .shared-config/install.sh --with-mike-macros"; exit 1; }' + - git fetch origin gh-pages --depth=1 2>/dev/null || true + - | + set -euo pipefail + if [ -f uv.lock ] || [ -f pyproject.toml ]; then + MIKE_LOCAL="$(uv run python mike_version.py local)" + uv run mike deploy "$MIKE_LOCAL" latest --update-aliases + uv run mike set-default latest + uv run mike list + else + MIKE_LOCAL="$(python mike_version.py local)" + mike deploy "$MIKE_LOCAL" latest --update-aliases + mike set-default latest + mike list + fi + echo "Local gh-pages updated. Run task mike:serve and open http://127.0.0.1:8000/" + + mike:deploy: + desc: >- + Push versioned docs to origin/gh-pages (mike). Set [versions] release in mkdocs/mike_versions.ini or export VERSION=… + dir: "{{.MKDOCS}}" + cmds: + - 'test -f mike_version.py || { echo "Run .shared-config/install.sh --with-mike-macros"; exit 1; }' + - git fetch origin gh-pages --depth=1 2>/dev/null || true + - | + set -euo pipefail + if [ -f uv.lock ] || [ -f pyproject.toml ]; then + MIKE_VERSION="$(uv run python mike_version.py release)" + uv run mike deploy --push --update-aliases "$MIKE_VERSION" latest + uv run mike set-default --push latest + uv run mike list + else + MIKE_VERSION="$(python mike_version.py release)" + mike deploy --push --update-aliases "$MIKE_VERSION" latest + mike set-default --push latest + mike list + fi + + mike:serve: + desc: Serve versioned site from local gh-pages (run mike:deploy-local first) + dir: "{{.MKDOCS}}" + cmds: + - | + if [ -f uv.lock ] || [ -f pyproject.toml ]; then + uv run mike serve -a 127.0.0.1:8000 + else + mike serve -a 127.0.0.1:8000 + fi diff --git a/.shared-config/assets/favicon.ico b/.shared-config/assets/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..4e15e04a3a347529c5fb567c6e94e1f7c2ae2693 GIT binary patch literal 2249 zcmcgu`9IX#A0EpvMi|RP3r&_>w5Y4wjP;|)kSHNw2h7s6h~myJQ7_wnLe zZeS1y;%shWe2pkDJ8bY6*9-r&7?9D-m_~|esi{ur=xLzg?OLx9Zylm_&)m^uQ1KUP z;wB5bWo*2aEqL<{5mcDNo=hTM3==fI$wWH&+z97J1RwNvmoHD*IxDFsu`_1&Dpv z?g4>)sqOpoAA2C-@}E*4IxAUa1jvZ^UF)k`06C`Uon zn{2o$4l6$nWZiT$MkcBR+KSrjsjjQv+_B$R`X&!NZ@&V(vf)}I3Ow?ihLPI?&f0PL){fgLq zebG8c*`Tjs6Gl^d5zYsc-Sh%E!yI9*!XduElbPqY*3c(>>p-1;V!mkH zePs7pLzs)cGO|0^cRL;_6NE<$Rq4_R?Detx$--RSFt8ukD_$jP?1(6DSrL-0|6=I? z`^^GmZP0$_1bUPV;3_(Ep>Ye_p`FJecNoCEtJ747?Nza6kcmb_ zOTwE%>!SB0&tJLl#*B{$q+ER9lAVZ5?c!KdX_KzW#;mZNM{S!D#f!C8BZ^!9Hum&+5o6a0Yq}DY*c#kju&_=@*Y53QGYjUeEQ={>Sk2m z8tBQzccVPYW}we?+1a6SENZFiRrh+6Z(BsIy8g49EBO~ay4Fr`ZtmbnHq2FvO(?Iu zZtKFmpdDY2T{@6=Rj#94c+(+Xs8>Uzqe6HSPk)iI(clu}5eHSD6R0|ugcxodV#v%^{+>{otF90|Q^)W;wd*W`(kXEH-}{C?Zl$+M1-ALDm^y7oJsy>}TA zh@eS-cFDSqY#ysVDSRet!++d;x|2eJwV=%wp(QO7!{1@lsSG+Jx|9!6r|9%kh5EL?nc!T<+tY%Dw~Un51vjAM?BBv=#Lb-tdj9`dTC9b%QrGUV*X?Wp9|dM#%Ahg zhkxHt+!Ju56nT;+{2j`eD>4!#5++N_`j#E2h$-5gz1G;<@4mJKgRklva8`?yup(I@ zB&Fh-Zk2XG$GS75I{(r$GT%xm3Exd);WOwDlmBFW++Ir8qx|hD&tE=-u=!*cy4>Ou zdtLnIfM1&Y!QC6dL#))hpC<$&hBXsVI<>6o##=vItMP1@b)ZYSS)}6^1rTzU+EfM>3IO3cR@!RIdV6#0-3&Zsb+s6G*tq^Uw zoUqWY98bY7h6bEL&R=TM)M#$VI#PYcT$$e)P#f-Je^86Mx*^JuDrJefyOCUh6 zOJux@!4MOBZ0bR(l*#7umQJZukL4bJkDX}-4oa15WcuDTsPjgMbc&gu4%1%>>>Rc3kpB0u8NvgyQO`Ay2?fMUFvo_ z2@~&GGF6&C)__AeRV+tfo-{MK+Va%s`Fq7R77uISCL3ESxKm?&_qp0|w1YvIm#xkO z$=uNr?CwtL*ig~P5#%}`Uk2=;A9)u=%z!^-KHc7@a|J9YpbQbFB47v#5DEGJ7VFXu XFx98m=!2{Lp9jd?)XJpjyes8j)^RuE literal 0 HcmV?d00001 diff --git a/.shared-config/assets/img/logo-dark.svg b/.shared-config/assets/img/logo-dark.svg new file mode 100644 index 0000000..00495ef --- /dev/null +++ b/.shared-config/assets/img/logo-dark.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/.shared-config/assets/img/logo-light.svg b/.shared-config/assets/img/logo-light.svg new file mode 100644 index 0000000..f6f57d7 --- /dev/null +++ b/.shared-config/assets/img/logo-light.svg @@ -0,0 +1,3 @@ + + + diff --git a/.shared-config/assets/logo.svg b/.shared-config/assets/logo.svg new file mode 100644 index 0000000..079955d --- /dev/null +++ b/.shared-config/assets/logo.svg @@ -0,0 +1,6 @@ + + + +network + + \ No newline at end of file diff --git a/.shared-config/codespellrc b/.shared-config/codespellrc new file mode 100644 index 0000000..4a007e3 --- /dev/null +++ b/.shared-config/codespellrc @@ -0,0 +1,6 @@ +[codespell] +skip = .git,.build,node_modules,*.png,*.jpg,*.jpeg,*.svg,*.ico,*.drawio,site,.venv +ignore-words-list = agntcy,authn,authz,github,localhost,mkdocs,pymdown,repo,repos,tls,url,urls,uri,uris,uuid,uuids,metadata,namespace,cli,api,apis,json,yaml,workflow,workflows +count = +check-filenames = +check-hidden = diff --git a/.shared-config/github/workflows/cicd.yml b/.shared-config/github/workflows/cicd.yml new file mode 100644 index 0000000..4eef836 --- /dev/null +++ b/.shared-config/github/workflows/cicd.yml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Cisco and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +# +# CI: build docs on PR/push when docs or mkdocs change. Copy to .github/workflows/ at repo root. + +name: CI/CD Pipeline + +on: + push: + tags: + - 'v*.*.*' + paths: + - 'docs/**' + - 'mkdocs/**' + - '.shared-config/**' + + pull_request: + paths: + - 'docs/**' + - 'mkdocs/**' + - '.shared-config/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + docs: + name: Docs + uses: ./.github/workflows/reusable-docs.yml + permissions: + contents: read + with: + version: dev + + success: + name: Success + if: ${{ !cancelled() && !contains(needs.*.result, 'cancelled') && !contains(needs.*.result, 'failure') }} + needs: + - docs + runs-on: ubuntu-latest + steps: + - name: Echo Success + run: echo "::notice Success!" diff --git a/.shared-config/github/workflows/docs-release.yml b/.shared-config/github/workflows/docs-release.yml new file mode 100644 index 0000000..ad3d08e --- /dev/null +++ b/.shared-config/github/workflows/docs-release.yml @@ -0,0 +1,229 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Cisco and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +# +# Ordered release pipeline: +# 1. Build docs — validate MkDocs output +# 2. Mike — push versioned site to gh-pages (from mkdocs/mike_versions.ini [versions] release) +# 3. Git tag — create v* tag on the commit that was published +# 4. Finalize — optional GitHub Release +# +# Copy to .github/workflows/ at repo root. Requires .shared-config/, mkdocs/, and mkdocs/pyproject.toml or uv.lock. + +name: Docs release + +on: + workflow_dispatch: + inputs: + create_github_release: + description: 'After tagging, create a GitHub Release for the same tag' + type: choice + options: + - 'no' + - 'yes' + default: 'no' + +permissions: + contents: write + +concurrency: + group: docs-release + cancel-in-progress: false + +jobs: + build: + name: '1. Build docs' + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + - name: Setup Taskfile + shell: bash + run: sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin + + - name: Setup UV + shell: bash + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Update PATH + shell: bash + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install AGNTCY theme into docs/ (shared-config) + shell: bash + run: | + chmod +x .shared-config/install.sh + ./.shared-config/install.sh + ./.shared-config/install.sh --with-mike-macros + + - name: Sync Python environment (mkdocs) + shell: bash + run: | + if [ -f mkdocs/pyproject.toml ] || [ -f mkdocs/uv.lock ]; then + (cd mkdocs && uv sync) + else + python3 -m pip install -r .shared-config/requirements-agntcy-docs-theme.txt + fi + + - name: Build + shell: bash + run: | + if [ -f mkdocs/uv.lock ] || [ -f mkdocs/pyproject.toml ]; then + export VERSION="$(cd mkdocs && uv run python mike_version.py release)" + else + export VERSION="$(cd mkdocs && python mike_version.py release)" + fi + task -t .shared-config/Taskfile.yml build + + mike: + name: '2. Mike (gh-pages)' + needs: build + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + - name: Setup Taskfile + shell: bash + run: sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin + + - name: Setup UV + shell: bash + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Update PATH + shell: bash + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install AGNTCY theme into docs/ (shared-config) + shell: bash + run: | + chmod +x .shared-config/install.sh + ./.shared-config/install.sh + ./.shared-config/install.sh --with-mike-macros + + - name: Sync Python environment (mkdocs) + shell: bash + run: | + if [ -f mkdocs/pyproject.toml ] || [ -f mkdocs/uv.lock ]; then + (cd mkdocs && uv sync) + else + python3 -m pip install -r .shared-config/requirements-agntcy-docs-theme.txt + fi + + - name: Configure Git for mike + run: | + git config user.name github-actions[bot] + git config user.email github-actions[bot]@users.noreply.github.com + + - name: Fetch gh-pages + continue-on-error: true + run: git fetch origin gh-pages --depth=1 + + - name: Mike deploy + shell: bash + run: task -t .shared-config/Taskfile.yml mike:deploy + + git-tag: + name: '3. Git tag' + needs: mike + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + - name: Setup UV + shell: bash + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Update PATH + shell: bash + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Sync Python environment (mkdocs) + shell: bash + run: | + if [ -f mkdocs/pyproject.toml ] || [ -f mkdocs/uv.lock ]; then + (cd mkdocs && uv sync) + else + python3 -m pip install -r .shared-config/requirements-agntcy-docs-theme.txt + fi + + - name: Create and push tag + shell: bash + env: + TAG_SHA: ${{ github.sha }} + run: | + set -euo pipefail + if [ -f mkdocs/uv.lock ] || [ -f mkdocs/pyproject.toml ]; then + VER="$(cd mkdocs && uv run python mike_version.py release)" + else + VER="$(cd mkdocs && python mike_version.py release)" + fi + TAG="v${VER}" + if git ls-remote --tags origin "refs/tags/${TAG}" | grep -q .; then + echo "::error::Tag ${TAG} already exists on origin. Bump [versions] release in mkdocs/mike_versions.ini or delete the remote tag." + exit 1 + fi + git tag -a "${TAG}" -m "Documentation release ${TAG}" "${TAG_SHA}" + git push origin "${TAG}" + + finalize: + name: '4. Deploy / release' + needs: git-tag + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup UV + shell: bash + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Update PATH + shell: bash + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Sync Python environment (mkdocs) + shell: bash + run: | + if [ -f mkdocs/pyproject.toml ] || [ -f mkdocs/uv.lock ]; then + (cd mkdocs && uv sync) + else + python3 -m pip install -r .shared-config/requirements-agntcy-docs-theme.txt + fi + + - name: GitHub Release + if: ${{ github.event.inputs.create_github_release == 'yes' }} + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + if [ -f mkdocs/uv.lock ] || [ -f mkdocs/pyproject.toml ]; then + VER="$(cd mkdocs && uv run python mike_version.py release)" + else + VER="$(cd mkdocs && python mike_version.py release)" + fi + TAG="v${VER}" + gh release create "${TAG}" --title "Documentation ${TAG}" --notes "Published from workflow [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}). Site is served from the gh-pages branch." + + - name: Pages note + if: ${{ github.event.inputs.create_github_release != 'yes' }} + run: | + echo "::notice::GitHub Pages (if configured from gh-pages) is already serving the mike version from step 2. Set **create_github_release** to **yes** to open a GitHub Release on the new tag." diff --git a/.shared-config/github/workflows/links.yml b/.shared-config/github/workflows/links.yml new file mode 100644 index 0000000..aa16453 --- /dev/null +++ b/.shared-config/github/workflows/links.yml @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Cisco and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +# +# Scheduled / manual full-site link check on built HTML. Copy to .github/workflows/ at repo root. + +name: Links + +on: + repository_dispatch: + workflow_dispatch: + schedule: + - cron: "00 06 * * *" + +jobs: + linkChecker: + runs-on: ubuntu-latest + permissions: + issues: write + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version: '1.23.1' + + - name: Setup Task + uses: arduino/setup-task@v2 + with: + version: '3.48.0' + + - name: Setup UV + shell: bash + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Update GITHUB_PATH + shell: bash + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install AGNTCY theme into docs/ (shared-config) + shell: bash + run: | + chmod +x .shared-config/install.sh + ./.shared-config/install.sh + ./.shared-config/install.sh --with-mike-macros + + - name: Sync Python environment (mkdocs) + shell: bash + run: | + if [ -f mkdocs/pyproject.toml ] || [ -f mkdocs/uv.lock ]; then + (cd mkdocs && uv sync) + else + python3 -m pip install -r .shared-config/requirements-agntcy-docs-theme.txt + fi + + - name: Build docs + shell: bash + run: task -t .shared-config/Taskfile.yml build + + - name: Link Checker + id: lychee + uses: lycheeverse/lychee-action@v2.0.2 + with: + args: '--config .shared-config/lychee.toml --exclude-path "(^|/)404\\.html$" .build/site' + fail: true diff --git a/.shared-config/github/workflows/pr-links.yml b/.shared-config/github/workflows/pr-links.yml new file mode 100644 index 0000000..f1d3eb6 --- /dev/null +++ b/.shared-config/github/workflows/pr-links.yml @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Cisco and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +# +# PR: lychee on changed docs pages only. Copy to .github/workflows/ at repo root. +# +# Optional: if your docs reference a fixed production URL, add a --remap line below +# (see https://lychee.cli.rs/recipes/remapping/) so those links resolve to .build/site. + +name: Check links in diffs + +on: + pull_request: + types: [opened, synchronize, reopened] + branches: [main, master] + workflow_dispatch: + +jobs: + check-links: + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + steps: + - name: Clone repository (PR) + if: github.event_name == 'pull_request' + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + submodules: recursive + + - name: Clone repository (manual) + if: github.event_name == 'workflow_dispatch' + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version: '1.23.1' + + - name: Setup Task + uses: arduino/setup-task@v2 + with: + version: '3.48.0' + + - name: Setup UV + shell: bash + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Update GITHUB_PATH + shell: bash + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install AGNTCY theme into docs/ (shared-config) + shell: bash + run: | + chmod +x .shared-config/install.sh + ./.shared-config/install.sh + ./.shared-config/install.sh --with-mike-macros + + - name: Sync Python environment (mkdocs) + shell: bash + run: | + if [ -f mkdocs/pyproject.toml ] || [ -f mkdocs/uv.lock ]; then + (cd mkdocs && uv sync) + else + python3 -m pip install -r .shared-config/requirements-agntcy-docs-theme.txt + fi + + - name: Build docs + shell: bash + run: task -t .shared-config/Taskfile.yml build + + - name: Fetch base branch + if: github.event_name == 'pull_request' + run: | + git remote add upstream https://github.com/${{ github.repository }}.git 2>/dev/null || true + git fetch upstream ${{ github.event.pull_request.base.ref }} + + - name: Get paths to check + id: get_paths + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "paths=.build/site" >> $GITHUB_OUTPUT + echo "has_paths=true" >> $GITHUB_OUTPUT + exit 0 + fi + BASE=$(git merge-base FETCH_HEAD HEAD) + changed=$(git diff --name-only $BASE...HEAD -- docs/) + paths="" + for f in $changed; do + [ -z "$f" ] && continue + case "$f" in *.md) ;; *) continue ;; esac + if [ "$f" = "docs/index.md" ]; then + paths="$paths .build/site/index.html" + else + path="${f#docs/}" + path="${path%.md}" + paths="$paths .build/site/${path}/" + fi + done + paths=$(echo $paths) + echo "paths=$paths" >> $GITHUB_OUTPUT + [ -n "$paths" ] && echo "has_paths=true" >> $GITHUB_OUTPUT || echo "has_paths=false" >> $GITHUB_OUTPUT + + - name: Remove paths lychee cannot parse + if: steps.get_paths.outputs.has_paths == 'true' + run: | + find .build/site -name '.index' -type f -delete + find .build/site -depth -name '.index' -type d -exec rm -rf {} \; + + - name: Check links + if: steps.get_paths.outputs.has_paths == 'true' + uses: lycheeverse/lychee-action@v2.0.2 + with: + lycheeVersion: "v0.22.0" + args: | + --no-progress + --config .shared-config/lychee.toml + --root-dir $PWD/.build/site + --exclude-all-private + ${{ steps.get_paths.outputs.paths }} + fail: true + output: lychee/out.md + + - name: Suggestions + if: failure() + run: | + echo -e "\nPlease review the links reported in the Check links step above." + echo -e "If a link is valid but fails due to a CAPTCHA challenge, IP blocking, login requirements, etc.," + echo -e "consider adding such links to .shared-config/lychee.toml exclude list to bypass future checks.\n" + exit 1 diff --git a/.shared-config/github/workflows/reusable-docs.yml b/.shared-config/github/workflows/reusable-docs.yml new file mode 100644 index 0000000..6194075 --- /dev/null +++ b/.shared-config/github/workflows/reusable-docs.yml @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Cisco and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +# +# Build-only documentation check. Copy to .github/workflows/ at repo root. +# Versioned publish (mike), tagging, and releases: docs-release.yml in this folder. + +name: Documentation + +on: + workflow_call: + inputs: + version: + description: 'MkDocs macros VERSION (docs_build_version)' + required: true + type: string + default: dev + + workflow_dispatch: + inputs: + version: + description: 'MkDocs macros VERSION' + required: false + type: string + default: dev + +permissions: + contents: read + +jobs: + docs: + name: Build docs + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + - name: Setup Taskfile + shell: bash + run: sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b ~/.local/bin + + - name: Setup UV + shell: bash + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Update GITHUB_PATH + shell: bash + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install AGNTCY theme into docs/ (shared-config) + shell: bash + run: | + chmod +x .shared-config/install.sh + ./.shared-config/install.sh + ./.shared-config/install.sh --with-mike-macros + + - name: Sync Python environment (mkdocs) + shell: bash + run: | + if [ -f mkdocs/pyproject.toml ] || [ -f mkdocs/uv.lock ]; then + (cd mkdocs && uv sync) + else + python3 -m pip install -r .shared-config/requirements-agntcy-docs-theme.txt + fi + + - name: Build docs + shell: bash + run: | + V="${{ inputs.version }}" + export VERSION="${V:-dev}" + task -t .shared-config/Taskfile.yml build diff --git a/.shared-config/install.sh b/.shared-config/install.sh new file mode 100755 index 0000000..6bbe883 --- /dev/null +++ b/.shared-config/install.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copy AGNTCY theme assets into docs/ (MkDocs serves extra_css and theme static files from docs_dir). +# Optional: ./install.sh --with-mike-macros → copies mike_version.py, mike_versions.ini, main.py into mkdocs/ +set -euo pipefail + +SHARED_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SHARED_ROOT/.." && pwd)" +DOCS="$REPO_ROOT/docs" +MKDOCS="$REPO_ROOT/mkdocs" + +mkdir -p "$DOCS/stylesheets" "$DOCS/assets/agntcy/img" +cp -f "$SHARED_ROOT/stylesheets/custom.css" "$DOCS/stylesheets/agntcy-docs.css" +cp -f "$SHARED_ROOT/assets/favicon.ico" "$DOCS/assets/agntcy/" +cp -f "$SHARED_ROOT/assets/img/"*.svg "$DOCS/assets/agntcy/img/" +if [[ -f "$SHARED_ROOT/assets/logo.svg" ]]; then + cp -f "$SHARED_ROOT/assets/logo.svg" "$DOCS/assets/agntcy/" +fi + +echo "AGNTCY docs theme installed under $DOCS (stylesheets/agntcy-docs.css, assets/agntcy/)." + +if [[ "${1:-}" == "--with-mike-macros" ]]; then + mkdir -p "$MKDOCS" + cp -f "$SHARED_ROOT/mkdocs/mike_version.py" "$MKDOCS/" + cp -f "$SHARED_ROOT/mkdocs/mike_versions.ini" "$MKDOCS/" + if [[ ! -f "$MKDOCS/main.py" ]]; then + cp -f "$SHARED_ROOT/mkdocs/main.py.example" "$MKDOCS/main.py" + echo "Created $MKDOCS/main.py from main.py.example" + else + echo "Leaving existing $MKDOCS/main.py unchanged." + fi + echo "Mike + macros helpers installed under $MKDOCS (mike_version.py, mike_versions.ini, main.py)." +fi diff --git a/.shared-config/lychee.toml b/.shared-config/lychee.toml new file mode 100644 index 0000000..3dcba3c --- /dev/null +++ b/.shared-config/lychee.toml @@ -0,0 +1,25 @@ +# https://github.com/lycheeverse/lychee +max_concurrency = 10 +timeout = 10 +accept = [200, 201, 204, 403, 429, 503] + +exclude = [ + 'localhost', + 'http://127\\.0\\.0\\.1', + 'https://127\\.0\\.0\\.1', + 'http://0\\.0\\.0\\.0', + 'host\\.docker\\.internal', + '\\.well-known', + 'example\\.com', + 'your-username', + 'company\\.com', + 'w3\\.org', + 'gnu\\.org', +] + +exclude_path = ['overrides', '\\.index(/|$)', '(^|/)404\\.html$'] +cache = true +max_retries = 3 +method = "get" +include_verbatim = false +user_agent = "lychee/agntcy-docs-theme" diff --git a/.shared-config/mkdocs/hooks.py b/.shared-config/mkdocs/hooks.py new file mode 100644 index 0000000..911276d --- /dev/null +++ b/.shared-config/mkdocs/hooks.py @@ -0,0 +1,13 @@ +"""MkDocs hooks to configure SSL certificates for include-markdown plugin.""" +import ssl +import certifi + + +def on_startup(**kwargs): + """Configure SSL context to use certifi certificates.""" + import urllib.request + + ssl_context = ssl.create_default_context(cafile=certifi.where()) + https_handler = urllib.request.HTTPSHandler(context=ssl_context) + opener = urllib.request.build_opener(https_handler) + urllib.request.install_opener(opener) diff --git a/.shared-config/mkdocs/main.py.example b/.shared-config/mkdocs/main.py.example new file mode 100644 index 0000000..923d4ef --- /dev/null +++ b/.shared-config/mkdocs/main.py.example @@ -0,0 +1,23 @@ +"""MkDocs macros: shared variables (from mkdocs.yml `extra`) and reusable helpers.""" + +from __future__ import annotations + +import html +import os + + +def define_env(env): + """Register variables and macros for mkdocs-macros-plugin.""" + + env.variables["docs_build_version"] = os.environ.get("VERSION", "dev") + + @env.macro + def var_tag(label: str, hint: str = "") -> str: + """Inline tag for env vars, flags, or keys. Use [[[ var_tag('MY_ENV') ]]] in Markdown.""" + safe_label = html.escape(str(label), quote=True) + safe_hint = html.escape(str(hint), quote=True) if hint else "" + title = f' title="{safe_hint}"' if safe_hint else "" + return ( + f'' + f"{safe_label}" + ) diff --git a/.shared-config/mkdocs/mike_version.py b/.shared-config/mkdocs/mike_version.py new file mode 100644 index 0000000..c9f70ee --- /dev/null +++ b/.shared-config/mkdocs/mike_version.py @@ -0,0 +1,46 @@ +"""Print mike version from mike_versions.ini; optional VERSION env overrides release.""" + +from __future__ import annotations + +import argparse +import configparser +import os +from pathlib import Path + + +def strip_leading_v(value: str) -> str: + value = value.strip() + return value[1:] if value.startswith("v") else value + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("local", "release")) + args = parser.parse_args() + + ini = Path(__file__).resolve().parent / "mike_versions.ini" + cfg = configparser.ConfigParser() + if not cfg.read(ini, encoding="utf-8"): + raise SystemExit(f"could not read {ini}") + + if args.mode == "local": + raw = cfg.get("versions", "local", fallback="dev") + out = strip_leading_v(raw) or "dev" + print(out) + return + + env = os.environ.get("VERSION", "").strip() + if env: + print(strip_leading_v(env)) + return + + raw = cfg.get("versions", "release", fallback="").strip() + if not raw: + raise SystemExit( + "Set [versions] release in mkdocs/mike_versions.ini or export VERSION=..." + ) + print(strip_leading_v(raw)) + + +if __name__ == "__main__": + main() diff --git a/.shared-config/mkdocs/mike_versions.ini b/.shared-config/mkdocs/mike_versions.ini new file mode 100644 index 0000000..5fe92eb --- /dev/null +++ b/.shared-config/mkdocs/mike_versions.ini @@ -0,0 +1,8 @@ +# Mike version labels (edit by hand). +# +# local — task mike:deploy-local (no git push) +# release — task mike:deploy; set before CI or export VERSION=v1.0.0 for one run. + +[versions] +local = dev +release = diff --git a/.shared-config/mkdocs/mkdocs.template.yml b/.shared-config/mkdocs/mkdocs.template.yml new file mode 100644 index 0000000..652a039 --- /dev/null +++ b/.shared-config/mkdocs/mkdocs.template.yml @@ -0,0 +1,152 @@ +# AGNTCY MkDocs starter — copy to mkdocs/mkdocs.yml and replace placeholders. +# Run ./.shared-config/install.sh (and --with-mike-macros if using mike + macros). +# +# Parity with agntcy/docs: hooks, redirects, awesome-pages, macros, mkdocstrings, +# swagger-ui-tag, include-markdown, mike, analytics + consent (replace GA property). + +site_name: MY_PROJECT_DOCS +site_url: https://example.org/ + +docs_dir: ../docs + +repo_name: org/repo +repo_url: https://github.com/org/repo +edit_uri: edit/main/docs/ + +extra: + version: + provider: mike + var: + docs_url: https://example.org/ + repo_url: https://github.com/org/repo + repo_slug: org/repo + org: AGNTCY + + copyright: > + © Copyright AGNTCY Contributors. + Change cookie settings + analytics: + provider: google + property: G-XXXXXXXXXX + consent: + title: Cookie consent + description: >- + We use cookies to recognize your repeated visits and preferences, as well + as to measure the effectiveness of our documentation and whether users + find what they're searching for. With your consent, you're helping us to + make our documentation better. + actions: + - accept + - reject + +extra_css: + - stylesheets/agntcy-docs.css + +markdown_extensions: + - admonition + - attr_list + - def_list + - md_in_html + - pymdownx.details + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + +hooks: + - ../.shared-config/mkdocs/hooks.py + +plugins: + search: + mike: + alias_type: redirect + redirects: + redirect_maps: + 'how-to-guides/identity-quickstart/index.md': 'identity/identity-quickstart.md' + 'how-to-guides/agent-directory/index.md': 'dir/getting-started.md' + 'messaging/slim-core.md': 'slim/overview.md' + 'messaging/slim-a2a.md': 'slim/slim-a2a.md' + 'messaging/slim-authentication.md': 'slim/slim-authentication.md' + 'messaging/slim-controller.md': 'slim/slim-controller.md' + 'messaging/slim-data-plane-config.md': 'slim/slim-data-plane-config.md' + 'messaging/slim-data-plane.md': 'slim/slim-data-plane.md' + 'messaging/slim-group-tutorial.md': 'slim/slim-group-tutorial.md' + 'messaging/slim-group.md': 'slim/slim-group.md' + 'messaging/slim-howto.md': 'slim/slim-howto.md' + 'messaging/slim-mcp.md': 'slim/slim-mcp.md' + 'messaging/slim-rpc.md': 'slim/slim-rpc.md' + 'messaging/slim-session.md': 'slim/slim-session.md' + 'messaging/slim-slimrpc-compiler.md': 'slim/slim-slimrpc-compiler.md' + awesome-pages: + filename: ".index" + collapse_single_pages: true + strict: false + macros: + module_name: main + j2_variable_start_string: "[[[" + j2_variable_end_string: "]]]" + mkdocstrings: + handlers: + python: + options: + docstring_style: sphinx + docstring_section_style: list + extensions: + - griffe_pydantic: + schema: true + swagger-ui-tag: + include-markdown: + +theme: + name: material + custom_dir: ../.shared-config/mkdocs/overrides + + features: + - announce.dismiss + - content.action.edit + - content.action.view + - content.code.annotate + - content.code.copy + - content.tabs.link + - content.tooltips + - navigation.footer + - navigation.indexes + - navigation.top + - navigation.tracking + - search.highlight + - search.share + - search.suggest + - toc.follow + + palette: + - media: "(prefers-color-scheme)" + toggle: + icon: material/link + name: Switch to light mode + - media: "(prefers-color-scheme: light)" + scheme: default + primary: indigo + accent: indigo + toggle: + icon: material/toggle-switch + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: black + accent: indigo + toggle: + icon: material/toggle-switch-off + name: Switch to system preference + font: + text: Roboto + code: Roboto Mono + favicon: assets/agntcy/favicon.ico + logo_light: assets/agntcy/img/logo-light.svg + logo_dark: assets/agntcy/img/logo-dark.svg diff --git a/.shared-config/mkdocs/overrides/partials/copyright.html b/.shared-config/mkdocs/overrides/partials/copyright.html new file mode 100644 index 0000000..bdf2dca --- /dev/null +++ b/.shared-config/mkdocs/overrides/partials/copyright.html @@ -0,0 +1,3 @@ +
+ {{ config.extra.copyright }} +
diff --git a/.shared-config/mkdocs/overrides/partials/logo.html b/.shared-config/mkdocs/overrides/partials/logo.html new file mode 100644 index 0000000..6187a5a --- /dev/null +++ b/.shared-config/mkdocs/overrides/partials/logo.html @@ -0,0 +1,2 @@ +logo +logo diff --git a/.shared-config/pymarkdown.yaml b/.shared-config/pymarkdown.yaml new file mode 100644 index 0000000..0d6f42d --- /dev/null +++ b/.shared-config/pymarkdown.yaml @@ -0,0 +1,28 @@ +plugins: + md013: + enabled: false + line_length: 120 + code_blocks: false + tables: false + md024: + enabled: true + siblings_only: true + md030: + enabled: false + md031: + enabled: false + md033: + enabled: false + allowed_elements: [details, summary, img, br, sub, sup, div] + md041: + enabled: false + md046: + enabled: false + md051: + enabled: false + md007: + enabled: false + md034: + enabled: true + md036: + enabled: false diff --git a/.shared-config/requirements-agntcy-docs-lint.txt b/.shared-config/requirements-agntcy-docs-lint.txt new file mode 100644 index 0000000..a286ea6 --- /dev/null +++ b/.shared-config/requirements-agntcy-docs-lint.txt @@ -0,0 +1,2 @@ +codespell>=2.2 +pymarkdownlnt>=0.9 diff --git a/.shared-config/requirements-agntcy-docs-theme.txt b/.shared-config/requirements-agntcy-docs-theme.txt new file mode 100644 index 0000000..7d39172 --- /dev/null +++ b/.shared-config/requirements-agntcy-docs-theme.txt @@ -0,0 +1,18 @@ +# Python deps for mkdocs/mkdocs.template.yml (Material, mike, macros, redirects, nav, API docs, swagger, includes). +mkdocs>=1.6 +mkdocs-material>=9.5 +pymdown-extensions>=9.4 +markdown>=3.2 +mkdocs-material-extensions>=1.0.3 +certifi>=2024.0.0 +mike>=2.1 +mkdocs-macros-plugin>=1.5 +mkdocs-redirects>=1.2 +mkdocs-awesome-pages-plugin>=2.10 +mkdocs-autorefs>=1.4 +mkdocstrings-python>=1.16 +griffe>=1.7 +griffe-pydantic>=1.1 +mkdocs-swagger-ui-tag>=0.7 +mkdocs-include-markdown-plugin>=7.2 +pygments>=2.12 diff --git a/.shared-config/stylesheets/custom.css b/.shared-config/stylesheets/custom.css new file mode 100644 index 0000000..9f1cbfd --- /dev/null +++ b/.shared-config/stylesheets/custom.css @@ -0,0 +1,181 @@ +[data-md-color-scheme="default"] { + --md-primary-fg-color: rgb(243, 246, 253); + --md-primary-bg-color: rgb(2, 81, 175); + --md-primary-fg-color--dark: rgb(243, 246, 253); + + --md-accent-fg-color: rgb(2, 81, 175); + + --md-default-fg-color: rgb(28, 30, 33); + --md-default-fg-color--light: rgb(28, 30, 33); + --md-default-fg-color--lighter: #00000052; + --md-default-fg-color--lightest: #00000012; + + --md-default-bg-color: rgb(239, 243, 252); + --md-default-bg-color--light: #e8eefb; + --md-default-bg-color--lighter: #eff3fc; + --md-default-bg-color--lightest: #f5f8fd; + + --md-footer-fg-color: rgb(2, 81, 175); + --md-footer-fg-color--light: rgb(28, 30, 33); + --md-footer-fg-color--lighter: rgb(28, 30, 33); + --md-footer-bg-color: rgb(232, 238, 251); + --md-footer-bg-color--dark: #00000052; + --md-typeset-a-color: rgb(2, 81, 175); + + --md-typeset-table-color--light: #e8eefb; +} + +[data-md-color-scheme="slate"] { + --md-primary-fg-color: rgb(3, 20, 43); + --md-primary-bg-color: rgb(251, 175, 70); + + --md-accent-fg-color: rgb(251, 175, 70); + + --md-default-fg-color: rgb(227, 227, 227); + --md-default-fg-color--light: rgb(227, 227, 227); + --md-default-fg-color--lighter: rgb(237, 237, 237); + --md-default-fg-color--lightest: rgb(3, 20, 43); + + --md-default-bg-color: rgb(3, 20, 43); + --md-code-bg-color: rgb(0, 42, 77); + --md-typeset-color: rgb(227, 227, 227); + + --md-footer-fg-color: rgb(251, 175, 70); + --md-footer-fg-color--light: rgb(227, 227, 227); + --md-footer-fg-color--lighter: rgb(227, 227, 227); + --md-footer-bg-color: rgb(3, 20, 43); + --md-footer-bg-color--dark: #00000052; + --md-typeset-a-color: rgb(2, 81, 175); + + --md-typeset-a-color: rgb(251, 175, 70) !important; + --md-typeset-a-color--hover: rgb(251, 175, 70) !important; + + --md-footer-meta-bg-color: rgb(232, 238, 251); + + --md-typeset-table-color--light: rgb(0, 42, 77); + + --md-code-fg-color: rgb(248, 248, 242); +} + +[data-md-color-primary=black] .md-header { + background-color: var(--md-primary-fg-color) !important; +} + +[data-md-color-scheme="default"] .md-search__form { + background-color: rgb(236, 237, 240); + color: var(--md-primary-bg-color); +} + +[data-md-color-scheme="default"] .md-search__form:hover { + background-color: rgb(236, 237, 240); +} + +[data-md-color-scheme="slate"] .md-search__form { + background-color: rgb(0, 42, 77); + color: var(--md-primary-bg-color) !important; +} + +[data-md-color-scheme="slate"] .md-search__form:hover { + background-color: rgb(0, 42, 77); +} + +[data-md-color-scheme="default"] .md-search__input::placeholder { + color: #9ea2a8; +} + +[data-md-color-scheme="slate"] .md-search__input::placeholder { + color: #59616b; +} + +.md-footer { + box-shadow: 0 0 .2rem #0000001a, 0 .2rem .4rem #0003; + background-color: var(--md-footer-bg-color); + color: var(--md-default-fg-color); +} + +.md-footer-meta { + background-color: var(--md-footer-bg-color); +} + +.md-footer__link { + color: var(--md-default-fg-color); +} + +.md-footer__link:hover { + color: var(--md-typeset-a-color); +} + +html [data-md-color-primary=black] .md-nav--primary .md-nav__title[for=__drawer] { + background-color: var(--md-default-bg-color); +} + +[data-md-color-scheme="default"] .logo-dark { + display: none !important; +} + +[data-md-color-scheme="slate"] .logo-light { + display: none !important; +} + +.md-sidebar--primary .md-nav__link--active { + font-weight: bold; + border: 0px solid transparent; + border-radius: 0.25rem; +} + +.md-sidebar--primary .md-nav__link { + padding-left: 10px; + padding-top: 5px; + padding-bottom: 5px; +} + +[data-md-color-scheme="default"] .md-sidebar--primary .md-nav__link--active { + background-color: rgb(220, 226, 238); +} + +[data-md-color-scheme="slate"] .md-sidebar--primary .md-nav__link--active { + background-color: rgb(12, 31, 54); +} + +.md-nav__title { + font-size: 0 !important; +} + +.md-nav__title img { + font-size: initial !important; +} + +/* Hide the duplicate site name in the header; do NOT hide the whole first + .md-header__topic — Material appends the mike version selector there. */ +.md-header__ellipsis > .md-header__topic:first-child > .md-ellipsis { + display: none !important; +} + +[data-md-color-scheme="default"] code { + background-color: rgb(246, 248, 250) !important; +} + +/* mkdocs-macros: {{ var_tag('NAME') }} */ +.doc-var-tag { + display: inline-flex; + align-items: center; + margin: 0 0.15em; + vertical-align: baseline; + border-radius: 0.25rem; + padding: 0.05em 0.35em; + font-size: 0.92em; + line-height: 1.35; + border: 1px solid var(--md-default-fg-color--lighter); + background-color: var(--md-default-bg-color--lightest); +} + +.doc-var-tag code { + background-color: transparent !important; + padding: 0 !important; + font-size: inherit !important; +} + +[data-md-color-scheme="slate"] .doc-var-tag { + border-color: var(--md-default-fg-color--lightest); + background-color: var(--md-code-bg-color); +} \ No newline at end of file From 1935f981a004edd6fcc34fb96eceba0b9a3117d4 Mon Sep 17 00:00:00 2001 From: Aron Kerekes Date: Thu, 2 Apr 2026 16:58:29 +0200 Subject: [PATCH 08/11] updating templates Signed-off-by: Aron Kerekes --- .shared-config/README.md | 1 + .shared-config/docs-stub/getting-started.md | 5 + .shared-config/docs-stub/index.md | 5 + .shared-config/install.sh | 29 ++- .shared-config/mkdocs/mkdocs.starter.yml | 119 ++++++++++++ .../requirements-agntcy-docs-theme.txt | 2 +- docs/assets/agntcy/favicon.ico | Bin 0 -> 2249 bytes docs/assets/agntcy/img/logo-dark.svg | 5 + docs/assets/agntcy/img/logo-light.svg | 3 + docs/assets/agntcy/logo.svg | 6 + docs/stylesheets/agntcy-docs.css | 181 ++++++++++++++++++ mkdocs/pyproject.toml | 2 +- mkdocs/uv.lock | 2 +- 13 files changed, 355 insertions(+), 5 deletions(-) create mode 100644 .shared-config/docs-stub/getting-started.md create mode 100644 .shared-config/docs-stub/index.md create mode 100644 .shared-config/mkdocs/mkdocs.starter.yml create mode 100644 docs/assets/agntcy/favicon.ico create mode 100644 docs/assets/agntcy/img/logo-dark.svg create mode 100644 docs/assets/agntcy/img/logo-light.svg create mode 100644 docs/assets/agntcy/logo.svg create mode 100644 docs/stylesheets/agntcy-docs.css diff --git a/.shared-config/README.md b/.shared-config/README.md index d0ece88..3b45695 100644 --- a/.shared-config/README.md +++ b/.shared-config/README.md @@ -114,6 +114,7 @@ Re-run after updating `.shared-config` from upstream. ``` 3. Copy `.shared-config/mkdocs/mkdocs.template.yml` to `mkdocs/mkdocs.yml` and set `site_name`, `site_url`, `repo_*`, `edit_uri`, `extra.var`, `extra.analytics.property`, and edit `plugins.redirects` as needed. Remove plugins you do not use (for example `mkdocstrings` if you have no API pages). + 4. Install deps and serve: ```bash diff --git a/.shared-config/docs-stub/getting-started.md b/.shared-config/docs-stub/getting-started.md new file mode 100644 index 0000000..c4eaf08 --- /dev/null +++ b/.shared-config/docs-stub/getting-started.md @@ -0,0 +1,5 @@ +# Getting started + +This is a placeholder page so MkDocs can build with an empty tree. + +Add your real guides here, or delete this file and adjust navigation (for example `.index` files with [awesome-pages](https://github.com/lukasgeiter/mkdocs-awesome-pages-plugin)). diff --git a/.shared-config/docs-stub/index.md b/.shared-config/docs-stub/index.md new file mode 100644 index 0000000..e4e9655 --- /dev/null +++ b/.shared-config/docs-stub/index.md @@ -0,0 +1,5 @@ +# Welcome + +This is a placeholder home page. Replace this file with your project documentation. + +After editing, remove or update any dummy pages added by `./.shared-config/install.sh`. diff --git a/.shared-config/install.sh b/.shared-config/install.sh index 6bbe883..d65c982 100755 --- a/.shared-config/install.sh +++ b/.shared-config/install.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash # Copy AGNTCY theme assets into docs/ (MkDocs serves extra_css and theme static files from docs_dir). +# Bootstraps mkdocs/mkdocs.yml from mkdocs.starter.yml and placeholder docs when those files are missing. # Optional: ./install.sh --with-mike-macros → copies mike_version.py, mike_versions.ini, main.py into mkdocs/ set -euo pipefail @@ -8,7 +9,7 @@ REPO_ROOT="$(cd "$SHARED_ROOT/.." && pwd)" DOCS="$REPO_ROOT/docs" MKDOCS="$REPO_ROOT/mkdocs" -mkdir -p "$DOCS/stylesheets" "$DOCS/assets/agntcy/img" +mkdir -p "$DOCS/stylesheets" "$DOCS/assets/agntcy/img" "$MKDOCS" cp -f "$SHARED_ROOT/stylesheets/custom.css" "$DOCS/stylesheets/agntcy-docs.css" cp -f "$SHARED_ROOT/assets/favicon.ico" "$DOCS/assets/agntcy/" cp -f "$SHARED_ROOT/assets/img/"*.svg "$DOCS/assets/agntcy/img/" @@ -18,8 +19,32 @@ fi echo "AGNTCY docs theme installed under $DOCS (stylesheets/agntcy-docs.css, assets/agntcy/)." +created_mkdocs=0 +if [[ ! -f "$MKDOCS/mkdocs.yml" ]]; then + cp -f "$SHARED_ROOT/mkdocs/mkdocs.starter.yml" "$MKDOCS/mkdocs.yml" + created_mkdocs=1 + echo "Created $MKDOCS/mkdocs.yml from mkdocs.starter.yml (replace placeholders; use mkdocs.template.yml for full agntcy/docs parity)." +else + echo "Leaving existing $MKDOCS/mkdocs.yml unchanged." +fi + +# Placeholder pages only when bootstrapping a new mkdocs.yml (avoids adding stubs to mature docs trees). +if [[ "$created_mkdocs" -eq 1 ]]; then + if [[ ! -f "$DOCS/index.md" ]]; then + cp -f "$SHARED_ROOT/docs-stub/index.md" "$DOCS/index.md" + echo "Created $DOCS/index.md (placeholder)." + else + echo "Leaving existing $DOCS/index.md unchanged." + fi + if [[ ! -f "$DOCS/getting-started.md" ]]; then + cp -f "$SHARED_ROOT/docs-stub/getting-started.md" "$DOCS/getting-started.md" + echo "Created $DOCS/getting-started.md (placeholder)." + else + echo "Leaving existing $DOCS/getting-started.md unchanged." + fi +fi + if [[ "${1:-}" == "--with-mike-macros" ]]; then - mkdir -p "$MKDOCS" cp -f "$SHARED_ROOT/mkdocs/mike_version.py" "$MKDOCS/" cp -f "$SHARED_ROOT/mkdocs/mike_versions.ini" "$MKDOCS/" if [[ ! -f "$MKDOCS/main.py" ]]; then diff --git a/.shared-config/mkdocs/mkdocs.starter.yml b/.shared-config/mkdocs/mkdocs.starter.yml new file mode 100644 index 0000000..74dd642 --- /dev/null +++ b/.shared-config/mkdocs/mkdocs.starter.yml @@ -0,0 +1,119 @@ +# Minimal MkDocs config for a new repo (install.sh copies this to mkdocs/mkdocs.yml when missing). +# For full parity with agntcy/docs (redirects, macros, API docs, Swagger), use mkdocs.template.yml instead. + +site_name: MY_PROJECT_DOCS +site_url: https://example.org/ + +docs_dir: ../docs + +repo_name: org/repo +repo_url: https://github.com/org/repo +edit_uri: edit/main/docs/ + +extra: + version: + provider: mike + var: + docs_url: https://example.org/ + repo_url: https://github.com/org/repo + repo_slug: org/repo + org: AGNTCY + + copyright: > + © Copyright AGNTCY Contributors. + Change cookie settings + analytics: + provider: google + property: G-XXXXXXXXXX + consent: + title: Cookie consent + description: >- + We use cookies to recognize your repeated visits and preferences, as well + as to measure the effectiveness of our documentation and whether users + find what they are searching for. With your consent, you're helping us to + make our documentation better. + actions: + - accept + - reject + +extra_css: + - stylesheets/agntcy-docs.css + +markdown_extensions: + - admonition + - attr_list + - def_list + - md_in_html + - pymdownx.details + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + +hooks: + - ../.shared-config/mkdocs/hooks.py + +plugins: + search: + mike: + alias_type: redirect + redirects: + redirect_maps: {} + awesome-pages: + filename: ".index" + collapse_single_pages: true + strict: false + +theme: + name: material + custom_dir: ../.shared-config/mkdocs/overrides + + features: + - announce.dismiss + - content.action.edit + - content.action.view + - content.code.annotate + - content.code.copy + - content.tabs.link + - content.tooltips + - navigation.footer + - navigation.indexes + - navigation.top + - navigation.tracking + - search.highlight + - search.share + - search.suggest + - toc.follow + + palette: + - media: "(prefers-color-scheme)" + toggle: + icon: material/link + name: Switch to light mode + - media: "(prefers-color-scheme: light)" + scheme: default + primary: indigo + accent: indigo + toggle: + icon: material/toggle-switch + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: black + accent: indigo + toggle: + icon: material/toggle-switch-off + name: Switch to system preference + font: + text: Roboto + code: Roboto Mono + favicon: assets/agntcy/favicon.ico + logo_light: assets/agntcy/img/logo-light.svg + logo_dark: assets/agntcy/img/logo-dark.svg diff --git a/.shared-config/requirements-agntcy-docs-theme.txt b/.shared-config/requirements-agntcy-docs-theme.txt index 7d39172..93fb3b6 100644 --- a/.shared-config/requirements-agntcy-docs-theme.txt +++ b/.shared-config/requirements-agntcy-docs-theme.txt @@ -1,5 +1,5 @@ # Python deps for mkdocs/mkdocs.template.yml (Material, mike, macros, redirects, nav, API docs, swagger, includes). -mkdocs>=1.6 +mkdocs>=1.6,<2 mkdocs-material>=9.5 pymdown-extensions>=9.4 markdown>=3.2 diff --git a/docs/assets/agntcy/favicon.ico b/docs/assets/agntcy/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..4e15e04a3a347529c5fb567c6e94e1f7c2ae2693 GIT binary patch literal 2249 zcmcgu`9IX#A0EpvMi|RP3r&_>w5Y4wjP;|)kSHNw2h7s6h~myJQ7_wnLe zZeS1y;%shWe2pkDJ8bY6*9-r&7?9D-m_~|esi{ur=xLzg?OLx9Zylm_&)m^uQ1KUP z;wB5bWo*2aEqL<{5mcDNo=hTM3==fI$wWH&+z97J1RwNvmoHD*IxDFsu`_1&Dpv z?g4>)sqOpoAA2C-@}E*4IxAUa1jvZ^UF)k`06C`Uon zn{2o$4l6$nWZiT$MkcBR+KSrjsjjQv+_B$R`X&!NZ@&V(vf)}I3Ow?ihLPI?&f0PL){fgLq zebG8c*`Tjs6Gl^d5zYsc-Sh%E!yI9*!XduElbPqY*3c(>>p-1;V!mkH zePs7pLzs)cGO|0^cRL;_6NE<$Rq4_R?Detx$--RSFt8ukD_$jP?1(6DSrL-0|6=I? z`^^GmZP0$_1bUPV;3_(Ep>Ye_p`FJecNoCEtJ747?Nza6kcmb_ zOTwE%>!SB0&tJLl#*B{$q+ER9lAVZ5?c!KdX_KzW#;mZNM{S!D#f!C8BZ^!9Hum&+5o6a0Yq}DY*c#kju&_=@*Y53QGYjUeEQ={>Sk2m z8tBQzccVPYW}we?+1a6SENZFiRrh+6Z(BsIy8g49EBO~ay4Fr`ZtmbnHq2FvO(?Iu zZtKFmpdDY2T{@6=Rj#94c+(+Xs8>Uzqe6HSPk)iI(clu}5eHSD6R0|ugcxodV#v%^{+>{otF90|Q^)W;wd*W`(kXEH-}{C?Zl$+M1-ALDm^y7oJsy>}TA zh@eS-cFDSqY#ysVDSRet!++d;x|2eJwV=%wp(QO7!{1@lsSG+Jx|9!6r|9%kh5EL?nc!T<+tY%Dw~Un51vjAM?BBv=#Lb-tdj9`dTC9b%QrGUV*X?Wp9|dM#%Ahg zhkxHt+!Ju56nT;+{2j`eD>4!#5++N_`j#E2h$-5gz1G;<@4mJKgRklva8`?yup(I@ zB&Fh-Zk2XG$GS75I{(r$GT%xm3Exd);WOwDlmBFW++Ir8qx|hD&tE=-u=!*cy4>Ou zdtLnIfM1&Y!QC6dL#))hpC<$&hBXsVI<>6o##=vItMP1@b)ZYSS)}6^1rTzU+EfM>3IO3cR@!RIdV6#0-3&Zsb+s6G*tq^Uw zoUqWY98bY7h6bEL&R=TM)M#$VI#PYcT$$e)P#f-Je^86Mx*^JuDrJefyOCUh6 zOJux@!4MOBZ0bR(l*#7umQJZukL4bJkDX}-4oa15WcuDTsPjgMbc&gu4%1%>>>Rc3kpB0u8NvgyQO`Ay2?fMUFvo_ z2@~&GGF6&C)__AeRV+tfo-{MK+Va%s`Fq7R77uISCL3ESxKm?&_qp0|w1YvIm#xkO z$=uNr?CwtL*ig~P5#%}`Uk2=;A9)u=%z!^-KHc7@a|J9YpbQbFB47v#5DEGJ7VFXu XFx98m=!2{Lp9jd?)XJpjyes8j)^RuE literal 0 HcmV?d00001 diff --git a/docs/assets/agntcy/img/logo-dark.svg b/docs/assets/agntcy/img/logo-dark.svg new file mode 100644 index 0000000..00495ef --- /dev/null +++ b/docs/assets/agntcy/img/logo-dark.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/docs/assets/agntcy/img/logo-light.svg b/docs/assets/agntcy/img/logo-light.svg new file mode 100644 index 0000000..f6f57d7 --- /dev/null +++ b/docs/assets/agntcy/img/logo-light.svg @@ -0,0 +1,3 @@ + + + diff --git a/docs/assets/agntcy/logo.svg b/docs/assets/agntcy/logo.svg new file mode 100644 index 0000000..079955d --- /dev/null +++ b/docs/assets/agntcy/logo.svg @@ -0,0 +1,6 @@ + + + +network + + \ No newline at end of file diff --git a/docs/stylesheets/agntcy-docs.css b/docs/stylesheets/agntcy-docs.css new file mode 100644 index 0000000..9f1cbfd --- /dev/null +++ b/docs/stylesheets/agntcy-docs.css @@ -0,0 +1,181 @@ +[data-md-color-scheme="default"] { + --md-primary-fg-color: rgb(243, 246, 253); + --md-primary-bg-color: rgb(2, 81, 175); + --md-primary-fg-color--dark: rgb(243, 246, 253); + + --md-accent-fg-color: rgb(2, 81, 175); + + --md-default-fg-color: rgb(28, 30, 33); + --md-default-fg-color--light: rgb(28, 30, 33); + --md-default-fg-color--lighter: #00000052; + --md-default-fg-color--lightest: #00000012; + + --md-default-bg-color: rgb(239, 243, 252); + --md-default-bg-color--light: #e8eefb; + --md-default-bg-color--lighter: #eff3fc; + --md-default-bg-color--lightest: #f5f8fd; + + --md-footer-fg-color: rgb(2, 81, 175); + --md-footer-fg-color--light: rgb(28, 30, 33); + --md-footer-fg-color--lighter: rgb(28, 30, 33); + --md-footer-bg-color: rgb(232, 238, 251); + --md-footer-bg-color--dark: #00000052; + --md-typeset-a-color: rgb(2, 81, 175); + + --md-typeset-table-color--light: #e8eefb; +} + +[data-md-color-scheme="slate"] { + --md-primary-fg-color: rgb(3, 20, 43); + --md-primary-bg-color: rgb(251, 175, 70); + + --md-accent-fg-color: rgb(251, 175, 70); + + --md-default-fg-color: rgb(227, 227, 227); + --md-default-fg-color--light: rgb(227, 227, 227); + --md-default-fg-color--lighter: rgb(237, 237, 237); + --md-default-fg-color--lightest: rgb(3, 20, 43); + + --md-default-bg-color: rgb(3, 20, 43); + --md-code-bg-color: rgb(0, 42, 77); + --md-typeset-color: rgb(227, 227, 227); + + --md-footer-fg-color: rgb(251, 175, 70); + --md-footer-fg-color--light: rgb(227, 227, 227); + --md-footer-fg-color--lighter: rgb(227, 227, 227); + --md-footer-bg-color: rgb(3, 20, 43); + --md-footer-bg-color--dark: #00000052; + --md-typeset-a-color: rgb(2, 81, 175); + + --md-typeset-a-color: rgb(251, 175, 70) !important; + --md-typeset-a-color--hover: rgb(251, 175, 70) !important; + + --md-footer-meta-bg-color: rgb(232, 238, 251); + + --md-typeset-table-color--light: rgb(0, 42, 77); + + --md-code-fg-color: rgb(248, 248, 242); +} + +[data-md-color-primary=black] .md-header { + background-color: var(--md-primary-fg-color) !important; +} + +[data-md-color-scheme="default"] .md-search__form { + background-color: rgb(236, 237, 240); + color: var(--md-primary-bg-color); +} + +[data-md-color-scheme="default"] .md-search__form:hover { + background-color: rgb(236, 237, 240); +} + +[data-md-color-scheme="slate"] .md-search__form { + background-color: rgb(0, 42, 77); + color: var(--md-primary-bg-color) !important; +} + +[data-md-color-scheme="slate"] .md-search__form:hover { + background-color: rgb(0, 42, 77); +} + +[data-md-color-scheme="default"] .md-search__input::placeholder { + color: #9ea2a8; +} + +[data-md-color-scheme="slate"] .md-search__input::placeholder { + color: #59616b; +} + +.md-footer { + box-shadow: 0 0 .2rem #0000001a, 0 .2rem .4rem #0003; + background-color: var(--md-footer-bg-color); + color: var(--md-default-fg-color); +} + +.md-footer-meta { + background-color: var(--md-footer-bg-color); +} + +.md-footer__link { + color: var(--md-default-fg-color); +} + +.md-footer__link:hover { + color: var(--md-typeset-a-color); +} + +html [data-md-color-primary=black] .md-nav--primary .md-nav__title[for=__drawer] { + background-color: var(--md-default-bg-color); +} + +[data-md-color-scheme="default"] .logo-dark { + display: none !important; +} + +[data-md-color-scheme="slate"] .logo-light { + display: none !important; +} + +.md-sidebar--primary .md-nav__link--active { + font-weight: bold; + border: 0px solid transparent; + border-radius: 0.25rem; +} + +.md-sidebar--primary .md-nav__link { + padding-left: 10px; + padding-top: 5px; + padding-bottom: 5px; +} + +[data-md-color-scheme="default"] .md-sidebar--primary .md-nav__link--active { + background-color: rgb(220, 226, 238); +} + +[data-md-color-scheme="slate"] .md-sidebar--primary .md-nav__link--active { + background-color: rgb(12, 31, 54); +} + +.md-nav__title { + font-size: 0 !important; +} + +.md-nav__title img { + font-size: initial !important; +} + +/* Hide the duplicate site name in the header; do NOT hide the whole first + .md-header__topic — Material appends the mike version selector there. */ +.md-header__ellipsis > .md-header__topic:first-child > .md-ellipsis { + display: none !important; +} + +[data-md-color-scheme="default"] code { + background-color: rgb(246, 248, 250) !important; +} + +/* mkdocs-macros: {{ var_tag('NAME') }} */ +.doc-var-tag { + display: inline-flex; + align-items: center; + margin: 0 0.15em; + vertical-align: baseline; + border-radius: 0.25rem; + padding: 0.05em 0.35em; + font-size: 0.92em; + line-height: 1.35; + border: 1px solid var(--md-default-fg-color--lighter); + background-color: var(--md-default-bg-color--lightest); +} + +.doc-var-tag code { + background-color: transparent !important; + padding: 0 !important; + font-size: inherit !important; +} + +[data-md-color-scheme="slate"] .doc-var-tag { + border-color: var(--md-default-fg-color--lightest); + background-color: var(--md-code-bg-color); +} \ No newline at end of file diff --git a/mkdocs/pyproject.toml b/mkdocs/pyproject.toml index 02fc884..52f9ac2 100644 --- a/mkdocs/pyproject.toml +++ b/mkdocs/pyproject.toml @@ -9,7 +9,7 @@ dependencies = [ "mkdocs-exclude==1.0.2", "mkdocs-awesome-pages-plugin==2.10.1", "markdown>=3.2", - "mkdocs>=1.3.0", + "mkdocs>=1.3.0,<2", "mkdocs-material-extensions>=1.0.3", "mkdocs-autorefs==1.4.1", "pygments>=2.12", diff --git a/mkdocs/uv.lock b/mkdocs/uv.lock index 9bac532..2f7d9e2 100644 --- a/mkdocs/uv.lock +++ b/mkdocs/uv.lock @@ -82,7 +82,7 @@ requires-dist = [ { name = "llama-index-core", specifier = ">=0.12.38,<0.13" }, { name = "markdown", specifier = ">=3.2" }, { name = "mike", specifier = ">=2.1.4" }, - { name = "mkdocs", specifier = ">=1.3.0" }, + { name = "mkdocs", specifier = ">=1.3.0,<2" }, { name = "mkdocs-autorefs", specifier = "==1.4.1" }, { name = "mkdocs-awesome-pages-plugin", specifier = "==2.10.1" }, { name = "mkdocs-exclude", specifier = "==1.0.2" }, From 89e0e74d737942575e90f25d0b46f5bedc05fc79 Mon Sep 17 00:00:00 2001 From: Aron Kerekes Date: Tue, 21 Apr 2026 13:18:02 +0200 Subject: [PATCH 09/11] chore: moving content Signed-off-by: Aron Kerekes --- docs/.index | 6 +- docs/dir/.index | 20 - docs/dir/architecture.md | 39 - docs/dir/dir-api-reference.md | 3 - docs/dir/directory-cli-reference.md | 649 ------- docs/dir/directory-cli.md | 474 ----- docs/dir/directory-gui.md | 118 -- docs/dir/directory-mcp.md | 108 -- docs/dir/directory-sdk.md | 146 -- docs/dir/events.md | 505 ------ docs/dir/federation-profiles.md | 361 ---- docs/dir/federation-troubleshooting.md | 236 --- docs/dir/getting-started.md | 343 ---- docs/dir/overview.md | 144 -- docs/dir/partner-prod-federation.md | 302 ---- docs/dir/prod-deployment.md | 187 -- docs/dir/records-validation.md | 270 --- docs/dir/runtime.md | 315 ---- docs/dir/scenarios.md | 661 ------- docs/dir/trust-model.md | 291 --- docs/dir/validation.md | 198 -- docs/identity/.index | 24 - docs/identity/arch_diagrams.md | 54 - docs/identity/connecting_idp.md | 175 -- docs/identity/creating_identities.md | 103 -- docs/identity/creating_policies.md | 87 - docs/identity/credentials.md | 50 - docs/identity/flow.md | 304 ---- docs/identity/identifier_examples.md | 220 --- docs/identity/identifiers.md | 15 - docs/identity/identity-quickstart.md | 243 --- docs/identity/identity.md | 83 - docs/identity/identity_service.md | 119 -- docs/identity/identity_service_api.md | 3 - docs/identity/identity_service_api_access.md | 35 - .../identity/identity_service_contributing.md | 571 ------ docs/identity/identity_service_development.md | 363 ---- docs/identity/identity_service_protofiles.md | 3 - docs/identity/identity_service_sdk.md | 68 - docs/identity/identity_service_settings.md | 168 -- docs/identity/openapi.md | 3 - docs/identity/vc_agent_badge.md | 86 - docs/identity/vc_mcp.md | 51 - docs/identity/verify_identity.md | 54 - docs/oasf/.index | 11 - docs/oasf/agent-record-guide.md | 118 -- docs/oasf/contributing.md | 467 ----- docs/oasf/decoding.md | 73 - docs/oasf/oasf-sdk.md | 38 - docs/oasf/oasf-server.md | 119 -- docs/oasf/open-agentic-schema-framework.md | 78 - docs/oasf/translation.md | 73 - docs/oasf/validation-comparison.md | 109 -- docs/oasf/validation.md | 67 - docs/slim/.index | 21 - docs/slim/overview.md | 123 -- docs/slim/slim-a2a.md | 296 --- docs/slim/slim-authentication.md | 289 --- docs/slim/slim-controller-reference.md | 186 -- docs/slim/slim-controller.md | 417 ----- docs/slim/slim-data-plane-config.md | 1586 ----------------- docs/slim/slim-data-plane.md | 82 - docs/slim/slim-group-tutorial.md | 801 --------- docs/slim/slim-group.md | 168 -- docs/slim/slim-howto.md | 436 ----- docs/slim/slim-mcp.md | 1034 ----------- docs/slim/slim-otel.md | 433 ----- docs/slim/slim-rpc.md | 598 ------- docs/slim/slim-session.md | 398 ----- docs/slim/slim-slimrpc-compiler.md | 365 ---- 70 files changed, 1 insertion(+), 16643 deletions(-) delete mode 100644 docs/dir/.index delete mode 100644 docs/dir/architecture.md delete mode 100644 docs/dir/dir-api-reference.md delete mode 100644 docs/dir/directory-cli-reference.md delete mode 100644 docs/dir/directory-cli.md delete mode 100644 docs/dir/directory-gui.md delete mode 100644 docs/dir/directory-mcp.md delete mode 100644 docs/dir/directory-sdk.md delete mode 100644 docs/dir/events.md delete mode 100644 docs/dir/federation-profiles.md delete mode 100644 docs/dir/federation-troubleshooting.md delete mode 100644 docs/dir/getting-started.md delete mode 100644 docs/dir/overview.md delete mode 100644 docs/dir/partner-prod-federation.md delete mode 100644 docs/dir/prod-deployment.md delete mode 100644 docs/dir/records-validation.md delete mode 100644 docs/dir/runtime.md delete mode 100644 docs/dir/scenarios.md delete mode 100644 docs/dir/trust-model.md delete mode 100644 docs/dir/validation.md delete mode 100644 docs/identity/.index delete mode 100644 docs/identity/arch_diagrams.md delete mode 100644 docs/identity/connecting_idp.md delete mode 100644 docs/identity/creating_identities.md delete mode 100644 docs/identity/creating_policies.md delete mode 100644 docs/identity/credentials.md delete mode 100644 docs/identity/flow.md delete mode 100644 docs/identity/identifier_examples.md delete mode 100644 docs/identity/identifiers.md delete mode 100644 docs/identity/identity-quickstart.md delete mode 100644 docs/identity/identity.md delete mode 100644 docs/identity/identity_service.md delete mode 100644 docs/identity/identity_service_api.md delete mode 100644 docs/identity/identity_service_api_access.md delete mode 100644 docs/identity/identity_service_contributing.md delete mode 100644 docs/identity/identity_service_development.md delete mode 100644 docs/identity/identity_service_protofiles.md delete mode 100644 docs/identity/identity_service_sdk.md delete mode 100644 docs/identity/identity_service_settings.md delete mode 100644 docs/identity/openapi.md delete mode 100644 docs/identity/vc_agent_badge.md delete mode 100644 docs/identity/vc_mcp.md delete mode 100644 docs/identity/verify_identity.md delete mode 100644 docs/oasf/.index delete mode 100644 docs/oasf/agent-record-guide.md delete mode 100644 docs/oasf/contributing.md delete mode 100644 docs/oasf/decoding.md delete mode 100644 docs/oasf/oasf-sdk.md delete mode 100644 docs/oasf/oasf-server.md delete mode 100644 docs/oasf/open-agentic-schema-framework.md delete mode 100644 docs/oasf/translation.md delete mode 100644 docs/oasf/validation-comparison.md delete mode 100644 docs/oasf/validation.md delete mode 100644 docs/slim/.index delete mode 100644 docs/slim/overview.md delete mode 100644 docs/slim/slim-a2a.md delete mode 100644 docs/slim/slim-authentication.md delete mode 100644 docs/slim/slim-controller-reference.md delete mode 100644 docs/slim/slim-controller.md delete mode 100644 docs/slim/slim-data-plane-config.md delete mode 100644 docs/slim/slim-data-plane.md delete mode 100644 docs/slim/slim-group-tutorial.md delete mode 100644 docs/slim/slim-group.md delete mode 100644 docs/slim/slim-howto.md delete mode 100644 docs/slim/slim-mcp.md delete mode 100644 docs/slim/slim-otel.md delete mode 100644 docs/slim/slim-rpc.md delete mode 100644 docs/slim/slim-session.md delete mode 100644 docs/slim/slim-slimrpc-compiler.md diff --git a/docs/.index b/docs/.index index 2bb0357..0eeed0a 100644 --- a/docs/.index +++ b/docs/.index @@ -1,11 +1,7 @@ nav: - Introduction: index.md - - OASF: oasf - - Agent Directory Service: dir - - Secure Low-Latency Interactive Messaging (SLIM): slim - - Identity: identity + - Getting Started: coffee-agntcy - Observability and Evaluation: obs-and-eval - CSIT: csit - - Getting Started: coffee-agntcy - Glossary: glossary.md - How to Contribute: contributing.md diff --git a/docs/dir/.index b/docs/dir/.index deleted file mode 100644 index e4a81e1..0000000 --- a/docs/dir/.index +++ /dev/null @@ -1,20 +0,0 @@ -nav: - - Overview: overview.md - - Getting Started: getting-started.md - - Features and Usage Scenarios: scenarios.md - - Production Deployment: prod-deployment.md - - CLI Guide: directory-cli.md - - MCP Server: directory-mcp.md - - Federation: - - Running a Federated Directory Instance: partner-prod-federation.md - - Profiles: federation-profiles.md - - Best Practices and Troubleshooting: federation-troubleshooting.md - - Reference: - - CLI Reference: directory-cli-reference.md - - API Reference: dir-api-reference.md - - Architecture: architecture.md - - Components: - - Runtime Discovery: runtime.md - - Event Streaming: events.md - - Records and Validation: records-validation.md - - Trust Model: trust-model.md diff --git a/docs/dir/architecture.md b/docs/dir/architecture.md deleted file mode 100644 index d00facf..0000000 --- a/docs/dir/architecture.md +++ /dev/null @@ -1,39 +0,0 @@ -# Architecture - -The architecture of the Agent Directory Service (ADS) is designed to support a scalable, secure, and efficient way to manage and distribute agent directory records across a distributed network of servers. - -## Components - -- **Records**: The fundamental units of information in the ADS, representing individual agents and their capabilities. Each record is uniquely identified and contains metadata describing the agent's skills, attributes, and constraints. - -- **Distributed Hash Table (DHT)**: A decentralized storage system that enables efficient lookup and retrieval of directory records. The DHT maps agent skills to record identifiers, allowing for quick discovery of relevant agents based on their capabilities. - -- **Content Routing Protocol**: A set of rules and mechanisms for routing requests and responses between agents and directory servers. This protocol ensures that queries for agent capabilities are efficiently directed to the appropriate servers hosting the relevant records. - -- **Agent Directory Servers**: Nodes in the ADS network responsible for storing and managing directory records. These servers participate in the DHT and implement the content routing protocol to facilitate agent discovery. - -- **Clients**: Tools that allow developers to interact with the ADS, publish agent records, and search for agents based on some criteria. - -- **Runtime Discovery**: Components that watch container runtimes (Docker, Kubernetes) for workloads and provide a gRPC API for querying them. Enables dynamic discovery of agents running in containerized environments, with support for resolving A2A agent cards and OASF records from discovered workloads. - -- **Event Streaming**: Real-time notification system that publishes Directory events to message brokers, enabling subscribers to react to record changes, discoveries, and other Directory operations. - -- **Security and Trust Mechanisms**: Features that ensure the integrity and authenticity of directory records and nodes, including cryptographic signing, verification of claims, secure communication protocols, and access controls. - -## Principles - -- **Scalability**: The ADS is designed to handle a large number of servers and records, with the ability to scale horizontally by adding more directory servers to the network. - -- **Efficiency**: The content routing protocol and DHT structure are optimized for fast lookups and minimal latency in agent discovery. - -- **Decentralization**: By leveraging a DHT and distributed servers, the ADS avoids single points of failure and promotes resilience and fault tolerance. - -- **Extensibility**: The ADS is built with flexibility in mind, allowing for the addition of new features, protocols, and integrations as needed. - -- **Security**: The architecture incorporates robust security measures to protect against unauthorized access, data tampering, and other threats. - -- **Interoperability**: The ADS is designed to work seamlessly with other systems and protocols, enabling integration with various agent frameworks and platforms. - -## System Architecture Diagram - -![ADS System Architecture](../assets/ads-architecture.png) diff --git a/docs/dir/dir-api-reference.md b/docs/dir/dir-api-reference.md deleted file mode 100644 index 9c05c39..0000000 --- a/docs/dir/dir-api-reference.md +++ /dev/null @@ -1,3 +0,0 @@ -# Directory API Reference - -The Directory API reference is available at [buf.build/agntcy/dir](https://buf.build/agntcy/dir). \ No newline at end of file diff --git a/docs/dir/directory-cli-reference.md b/docs/dir/directory-cli-reference.md deleted file mode 100644 index 788f85b..0000000 --- a/docs/dir/directory-cli-reference.md +++ /dev/null @@ -1,649 +0,0 @@ -# Directory CLI Command Reference - -## Storage Operations - -### `dirctl push ` - -Stores records in the content-addressable store. Has the following features: - -- Supports OASF v1, v2, v3 record formats -- Content-addressable storage with CID generation -- Optional cryptographic signing -- Data integrity validation - -??? example - - ```bash - # Push from file - dirctl push agent-model.json - - # Push from stdin - cat agent-model.json | dirctl push --stdin - - # Push with signature - dirctl push agent-model.json --sign --key private.key - ``` - -### `dirctl pull ` - -Retrieves records by their Content Identifier (CID) or name reference. - -**Supported Reference Formats:** - -| Format | Description | -|--------|-------------| -| `` | Direct lookup by CID | -| `` | Retrieves the latest version | -| `:` | Retrieves the specified version | -| `@` | Hash-verified lookup (fails if resolved CID doesn't match) | -| `:@` | Hash-verified lookup for a specific version | - -??? example - - ```bash - # Pull by CID - dirctl pull baeareihdr6t7s6sr2q4zo456sza66eewqc7huzatyfgvoupaqyjw23ilvi - - # Pull by name (latest version) - dirctl pull cisco.com/agent - - # Pull by name with specific version - dirctl pull cisco.com/agent:v1.0.0 - - # Pull with hash verification - dirctl pull cisco.com/agent@bafyreib... - dirctl pull cisco.com/agent:v1.0.0@bafyreib... - - # Pull with signature verification - dirctl pull --signature --public-key public.key - ``` - -**Hash Verification:** - -The `@` suffix enables hash verification. This command fails if the resolved CID doesn't match the expected digest: - -```bash -# Succeeds if cisco.com/agent:v1.0.0 resolves to bafyreib... -dirctl pull cisco.com/agent:v1.0.0@bafyreib... - -# Fails with error if CIDs don't match -dirctl pull cisco.com/agent@wrong-cid -# Error: hash verification failed: resolved CID "bafyreib..." does not match expected digest "wrong-cid" -``` - -**Version Resolution:** - -When no version is specified, commands return the most recently created record (by record's `created_at` field). This allows non-semver tags like `latest`, `dev`, or `stable`. - -### `dirctl delete ` - -Removes records from storage. - -??? example - - ```bash - # Delete a record - dirctl delete baeareihdr6t7s6sr2q4zo456sza66eewqc7huzatyfgvoupaqyjw23ilvi - ``` - -### `dirctl info ` - -Displays metadata about stored records using CID or name reference. - -**Supported Reference Formats:** - -| Format | Description | -|--------|-------------| -| `` | Direct lookup by content address | -| `` | Displays the most recently created version | -| `:` | Displays the specified version | -| `@` | Hash-verified lookup | -| `:@` | Hash-verified lookup for a specific version | - -??? example - - ```bash - # Info by CID (existing) - dirctl info baeareihdr6t7s6sr2q4zo456sza66eewqc7huzatyfgvoupaqyjw23ilvi - - # Info by name (latest version) - dirctl info cisco.com/agent --output json - - # Info by name with specific version - dirctl info cisco.com/agent:v1.0.0 --output json - ``` - -## Import Operations - -Import records from external registries into DIR. Supports automated batch imports from various registry types. - -### `dirctl import [flags]` - -Fetch and import records from external registries. - -**Supported Registries:** - -- `mcp` - Model Context Protocol registry v0.1 - -**Configuration Options:** - -| Flag | Environment Variable | Description | Required | Default | -|------|---------------------|-------------|----------|---------| -| `--type` | - | Registry type (mcp, a2a) | Yes | - | -| `--url` | - | Registry base URL | Yes | - | -| `--filter` | - | Registry-specific filters (key=value, repeatable) | No | - | -| `--limit` | - | Maximum records to import (0 = no limit) | No | 0 | -| `--dry-run` | - | Preview without importing | No | false | -| `--debug` | - | Enable debug output (shows MCP source and OASF record for failures) | No | false | -| `--force` | - | Force reimport of existing records (skip deduplication) | No | false | -| `--enrich-config` | - | Path to MCPHost configuration file (mcphost.json) | No | importer/enricher/mcphost.json | -| `--enrich-skills-prompt` | - | Optional: path to custom skills prompt template or inline prompt | No | "" (uses default) | -| `--enrich-domains-prompt` | - | Optional: path to custom domains prompt template or inline prompt | No | "" (uses default) | -| `--enrich-rate-limit` | - | Maximum LLM API requests per minute (to avoid rate limit errors) | No | 10 | -| `--sign` | - | Sign records after pushing (uses OIDC by default) | No | false | -| `--key` | - | Path to private key file for signing (requires `--sign`) | No | - | -| `--oidc-token` | - | OIDC token for non-interactive signing (requires `--sign`) | No | - | -| `--fulcio-url` | - | Sigstore Fulcio URL (requires `--sign`) | No | https://fulcio.sigstore.dev | -| `--rekor-url` | - | Sigstore Rekor URL (requires `--sign`) | No | https://rekor.sigstore.dev | -| `--server-addr` | DIRECTORY_CLIENT_SERVER_ADDRESS | DIR server address | No | localhost:8888 | - -!!! note - - By default, the importer performs deduplication: it builds a cache of existing records (by name and version) and skips importing records that already exist. This prevents duplicate imports when running the import command multiple times. Use `--force` to bypass deduplication and reimport existing records. Use `--debug` to see detailed output including which records were skipped and why imports failed. - -??? example - - ```bash - # Import from MCP registry - dirctl import --type=mcp --url=https://registry.modelcontextprotocol.io/v0.1 - - # Import with debug output (shows detailed diagnostics for failures) - dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --debug - - # Force reimport of existing records (skips deduplication) - dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --force - - # Import with time-based filter - dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --filter=updated_since=2025-08-07T13:15:04.280Z - - # Combine multiple filters - dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --filter=search=github \ - --filter=version=latest \ - --filter=updated_since=2025-08-07T13:15:04.280Z - - # Limit number of records - dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --limit=50 - - # Preview without importing (dry run) - dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --dry-run - - # Import and sign records with OIDC (opens browser for authentication) - dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --sign - - # Import and sign records with a private key - dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --sign \ - --key=/path/to/cosign.key - - # Import with rate limiting for LLM API calls - dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --enrich-rate-limit=5 - ``` - -**MCP Registry Filters:** - -For the Model Context Protocol registry, available filters include: - -- `search` - Filter by server name (substring match) -- `version` - Filter by version ('latest' for latest version, or an exact version like '1.2.3') -- `updated_since` - Filter by updated time (RFC3339 datetime format, e.g., '2025-08-07T13:15:04.280Z') - -See the [MCP Registry API docs](https://registry.modelcontextprotocol.io/docs#/operations/list-servers#Query-Parameters) for the complete list of supported filters. - -### LLM-based Enrichment (Mandatory) - -**Enrichment is mandatory** — the import command automatically enriches MCP server records using LLM models to map them to appropriate OASF skills and domains. Records from the oasf-sdk translator are incomplete and require enrichment to be valid. This is powered by [mcphost](https://github.com/mark3labs/mcphost), which provides a Model Context Protocol (MCP) host that can run AI models with tool-calling capabilities. - -**Requirements:** - -- `dirctl` binary (includes the built-in MCP server with `agntcy_oasf_get_schema_skills` and `agntcy_oasf_get_schema_domains` tools) -- An LLM model with tool-calling support (GPT-4o, Claude, or compatible Ollama models) -- The `mcphost.json` configuration file must include the `dir-mcp-server` entry that runs `dirctl mcp serve`. This MCP server provides the schema tools needed for enrichment. - -**How it works:** - -1. The enricher starts an MCP server using `dirctl mcp serve` -2. The LLM uses the `agntcy_oasf_get_schema_skills` tool to browse available OASF skills -3. The LLM uses the `agntcy_oasf_get_schema_domains` tool to browse available OASF domains -4. Based on the MCP server description and capabilities, the LLM selects appropriate skills and domains -5. Selected skills and domains replace the defaults in the imported records - -**Setting up mcphost:** - -Edit a configuration file (default: `importer/enricher/mcphost.json`): - -```json -{ - "mcpServers": { - "dir-mcp-server": { - "command": "dirctl", - "args": ["mcp", "serve"], - "env": { - "OASF_API_VALIDATION_SCHEMA_URL": "https://schema.oasf.outshift.com" - } - } - }, - "model": "azure:gpt-4o", - "max-tokens": 4096, - "max-steps": 20 -} -``` - -**Recommended LLM providers:** - -- `azure:gpt-4o` - Azure OpenAI GPT-4o (recommended for speed and accuracy) -- `ollama:qwen3:8b` - Local Qwen3 via Ollama - -**Environment variables for LLM providers:** - -- Azure OpenAI: `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_DEPLOYMENT` - -**Customizing Enrichment Prompts:** - -The enricher uses separate default prompt templates for skills and domains. You can customize these prompts for specific use cases: - -- **Skills**: Use default (omit `--enrich-skills-prompt`), or `--enrich-skills-prompt=/path/to/custom-skills-prompt.md`, or `--enrich-skills-prompt="Your custom prompt text..."` -- **Domains**: Use default (omit `--enrich-domains-prompt`), or `--enrich-domains-prompt=/path/to/custom-domains-prompt.md`, or `--enrich-domains-prompt="Your custom prompt text..."` - -The default prompt templates are available at `importer/enricher/enricher.skills.prompt.md` and `importer/enricher/enricher.domains.prompt.md`. - -??? example "Import with custom enrichment" - - ```bash - # Import with custom mcphost configuration - dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --enrich-config=/path/to/custom-mcphost.json - - # Import with custom prompt templates - dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --enrich-skills-prompt=/path/to/custom-skills-prompt.md \ - --enrich-domains-prompt=/path/to/custom-domains-prompt.md \ - --debug - ``` - -### Signing Records During Import - -Records can be signed during import using the `--sign` flag. Signing options work the same as the standalone `dirctl sign` command (see [Security & Verification](#security-verification)). - -```bash -# Sign with OIDC (opens browser) -dirctl import --type=mcp --url=https://registry.modelcontextprotocol.io/v0.1 --sign - -# Sign with a private key -dirctl import --type=mcp --url=https://registry.modelcontextprotocol.io/v0.1 --sign --key=/path/to/cosign.key -``` - -### Rate Limiting for LLM API Calls - -When importing large batches of records, the enrichment process makes LLM API calls for each record. To avoid hitting rate limits from LLM providers, use the `--enrich-rate-limit` flag: - -```bash -# Import with reduced rate limit (5 requests per minute) -dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --enrich-rate-limit=5 - -# Import with higher rate limit for providers with generous limits -dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --enrich-rate-limit=30 -``` - -## Routing Operations - -The routing commands manage record announcement and discovery across the peer-to-peer network. - -### `dirctl routing publish ` - -Announces records to the network for discovery by other peers. The command does the following: - -- Announces record to DHT network. -- Makes record discoverable by other peers. -- Stores routing metadata locally. -- Enables network-wide discovery. - -??? example - - ```bash - # Publish a record to the network - dirctl routing publish baeareihdr6t7s6sr2q4zo456sza66eewqc7huzatyfgvoupaqyjw23ilvi - ``` - -### `dirctl routing unpublish ` - -Removes records from network discovery while keeping them in local storage. The command does the following: - -- Removes DHT announcements. -- Stops network discovery. -- Keeps record in local storage. -- Cleans up routing metadata. - -??? example - - ```bash - # Remove from network discovery - dirctl routing unpublish baeareihdr6t7s6sr2q4zo456sza66eewqc7huzatyfgvoupaqyjw23ilvi - ``` - -### `dirctl routing list [flags]` - -Queries local published records with optional filtering. - -The following flags are available: - -- `--skill ` - Filter by skill (repeatable) -- `--locator ` - Filter by locator type (repeatable) -- `--domain ` - Filter by domain (repeatable) -- `--module ` - Filter by module name (repeatable) -- `--cid ` - List specific record by CID -- `--limit ` - Limit number of results - -??? example - - ```bash - # List all local published records - dirctl routing list - - # List by skill - dirctl routing list --skill "AI" - dirctl routing list --skill "Natural Language Processing" - - # List by locator type - dirctl routing list --locator "docker-image" - - # List by module - dirctl routing list --module "runtime/framework" - - # Multiple criteria (AND logic) - dirctl routing list --skill "AI" --locator "docker-image" - dirctl routing list --domain "healthcare" --module "runtime/language" - - # Specific record by CID - dirctl routing list --cid baeareihdr6t7s6sr2q4zo456sza66eewqc7huzatyfgvoupaqyjw23ilvi - - # Limit results - dirctl routing list --skill "AI" --limit 5 - ``` - -### `dirctl routing search [flags]` - -Discovers records from other peers across the network. - -The following flags are available: - -- `--skill ` - Search by skill (repeatable) -- `--locator ` - Search by locator type (repeatable) -- `--domain ` - Search by domain (repeatable) -- `--module ` - Search by module name (repeatable) -- `--limit ` - Maximum results to return -- `--min-score ` - Minimum match score threshold - -The output includes the following: - -- Record CID and provider peer information -- Match score showing query relevance -- Specific queries that matched -- Peer connection details - -??? example - - ```bash - # Search for AI records across the network - dirctl routing search --skill "AI" - - # Search with multiple criteria - dirctl routing search --skill "AI" --skill "ML" --min-score 2 - - # Search by locator type - dirctl routing search --locator "docker-image" - - # Search by module - dirctl routing search --module "runtime/framework" - - # Advanced search with scoring - dirctl routing search --skill "web-development" --limit 10 --min-score 1 - dirctl routing search --domain "finance" --module "validation" --min-score 2 - ``` - -**Output includes:** - -### `dirctl routing info` - -Shows routing statistics and summary information. - -The output includes the following: - -- Total published records count -- Skills distribution with counts -- Locators distribution with counts -- Helpful usage tips - -??? example - - ```bash - # Show local routing statistics - dirctl routing info - ``` - -## Search & Discovery - -### `dirctl search [flags]` - -General content search across all records using the search service. - -The following flags are available: - -- `--query ` - Search criteria (repeatable) -- `--limit ` - Maximum results -- `--offset ` - Result offset for pagination - -??? example - - ```bash - # Search by record name - dirctl search --query "name=my-agent" - - # Search by version - dirctl search --query "version=v1.0.0" - - # Search by skill ID - dirctl search --query "skill-id=10201" - - # Complex search with multiple criteria - dirctl search --limit 10 --offset 0 \ - --query "name=my-agent" \ - --query "skill-name=Text Completion" \ - --query "locator=docker-image:https://example.com/image" - ``` - -## Security & Verification - -### Name Verification - -Record name verification proves that the signing key is authorized by the domain claimed in the record's name field. - -**Requirements:** - -- Record name must include a protocol prefix: `https://domain/path` or `http://domain/path` -- A JWKS file must be hosted at `:///.well-known/jwks.json` -- The record must be signed with the private key corresponding to a public key present in that JWKS file - -**Workflow:** - -1. Push a record with a verifiable name. - - ```bash - dirctl push record.json --output raw - # Returns: bafyreib... - ``` - -2. Sign the record (triggers automatic verification). - - ```bash - dirctl sign --key private.key - ``` - -3. Check verification status using [`dirctl naming verify`](./directory-cli.md#dirctl-naming-verify-reference). - -### `dirctl sign [flags]` - -Signs records for integrity and authenticity. When signing a record with a verifiable name (e.g., `https://domain/path`), the system automatically attempts to verify domain authorization via JWKS. See [Name Verification](#name-verification) for details. - -??? example - - ```bash - # Sign with private key - dirctl sign --key private.key - - # Sign with OIDC (keyless signing) - dirctl sign --oidc --fulcio-url https://fulcio.example.com - ``` - -### `dirctl naming verify ` - -Verifies that a record's signing key is authorized by the domain claimed in its name field. Checks if the signing key matches a public key in the domain's JWKS file hosted at `/.well-known/jwks.json`. - -**Supported Reference Formats:** - -| Format | Description | -|--------|-------------| -| `` | Verify by content address | -| `` | Verify the most recently created version | -| `:` | Verify a specific version | - -??? example - - ```bash - # Verify by CID - dirctl naming verify bafyreib... --output json - - # Verify by name (latest version) - dirctl naming verify cisco.com/agent --output json - - # Verify by name with specific version - dirctl naming verify cisco.com/agent:v1.0.0 --output json - ``` - - Example verification response: - - ```json - { - "cid": "bafyreib...", - "verified": true, - "domain": "cisco.com", - "method": "jwks", - "key_id": "key-1", - "verified_at": "2026-01-21T10:30:00Z" - } - ``` - -### `dirctl verify [flags]` - -Verifies record signatures. - -??? example - - ```bash - # Verify with public key - dirctl verify record.json signature.sig --key public.key - ``` - -### `dirctl validate [] [flags]` - -Validates OASF record JSON from a file or stdin against the OASF schema. The JSON can be provided as a file path or piped from stdin (e.g., from `dirctl pull`). A schema URL must be provided via `--url` for API-based validation. - -| Flag | Description | -|------|-------------| -| `--url ` | OASF schema URL for API-based validation (required) | - -??? example - - ```bash - # Validate a file with API-based validation - dirctl validate record.json --url https://schema.oasf.outshift.com - - # Validate JSON piped from stdin - cat record.json | dirctl validate --url https://schema.oasf.outshift.com - - # Validate a record pulled from directory - dirctl pull --output json | dirctl validate --url https://schema.oasf.outshift.com - - # Validate all records (using shell scripting) - for cid in $(dirctl search --output jsonl | jq -r '.record_cid'); do - dirctl pull "$cid" | dirctl validate --url https://schema.oasf.outshift.com - done - ``` - -## Synchronization - -### `dirctl sync create ` - -Creates peer-to-peer synchronization. - -??? example - - ```bash - # Create sync with remote peer - dirctl sync create https://peer.example.com - ``` - -### `dirctl sync list` - -Lists active synchronizations. - -??? example - - ```bash - # Show all active syncs - dirctl sync list - ``` - -### `dirctl sync status ` - -Checks synchronization status. - -??? example - - ```bash - # Check specific sync status - dirctl sync status abc123-def456-ghi789 - ``` - -### `dirctl sync delete ` - -Removes synchronization. - -??? example - - ```bash - # Delete a sync - dirctl sync delete abc123-def456-ghi789 - ``` diff --git a/docs/dir/directory-cli.md b/docs/dir/directory-cli.md deleted file mode 100644 index 5dc6cfd..0000000 --- a/docs/dir/directory-cli.md +++ /dev/null @@ -1,474 +0,0 @@ -# Directory CLI Guide - -The Directory CLI (`dirctl`) provides comprehensive command-line tools for interacting with the Directory system, including storage, routing, search, and security operations. - -This guide provides an overview of how to get started with the CLI, how to use its features, as well as common workflows and usage examples. - -For detailed reference information, see the [Directory CLI Reference](directory-cli-reference.md). - -## Installation - -The Directory CLI can be installed in the following ways: - -=== "Using Homebrew" - - ```bash - brew tap agntcy/dir https://github.com/agntcy/dir/ - brew install dirctl - ``` - -=== "Using Release Binaries" - - ```bash - # Download from GitHub Releases - curl -L https://github.com/agntcy/dir/releases/latest/download/dirctl-linux-amd64 -o dirctl - chmod +x dirctl - sudo mv dirctl /usr/local/bin/ - ``` - -=== "Using Source" - - ```bash - git clone https://github.com/agntcy/dir - cd dir - task build-dirctl - ``` - -=== "Using Container Image" - - ```bash - docker pull ghcr.io/agntcy/dir-ctl:latest - docker run --rm ghcr.io/agntcy/dir-ctl:latest --help - ``` - -## Quick Start - -The following example demonstrates how to store, publish, search, and retrieve a record using the Directory CLI: - -1. Store a record - - ```bash - dirctl push my-agent.json - ``` - - Returns: `baeareihdr6t7s6sr2q4zo456sza66eewqc7huzatyfgvoupaqyjw23ilvi` - -1. Publish for network discovery - - ```bash - dirctl routing publish baeareihdr6t7s6sr2q4zo456sza66eewqc7huzatyfgvoupaqyjw23ilvi - ``` - -1. Search for records - - ```bash - dirctl routing search --skill "AI" --limit 10 - ``` - -1. Retrieve a record - - ```bash - # Pull by CID - dirctl pull baeareihdr6t7s6sr2q4zo456sza66eewqc7huzatyfgvoupaqyjw23ilvi - - # Or pull by name (if the record has a verifiable name) - dirctl pull cisco.com/agent:v1.0.0 - ``` - -!!! note "Name-based References" - - The CLI supports Docker-style name references in addition to CIDs. Records can be pulled using formats like `name`, `name:version`, or `name:version@cid` for hash-verified lookups. See [Name Verification](#name-verification) for details. - -!!! note "Authentication for federation" - - When accessing Directory federation nodes, authenticate first with `dirctl auth login`. See [Authentication](#authentication) for details. - -## Common Workflows - -### Publishing Workflow - -The following workflow demonstrates how to publish a record to the network: - -1. Store your record - - ```bash - # Using --output raw for clean scripting - CID=$(dirctl push my-agent.json --output raw) - echo "Stored with CID: $CID" - ``` - -1. Publish for discovery - - ```bash - dirctl routing publish $CID - ``` - -1. Verify the record is published - - ```bash - # Use JSON output for programmatic verification - dirctl routing list --cid $CID --output json - ``` - -1. Check routing statistics - - ```bash - dirctl routing info - ``` - -### Discovery Workflow - -The following workflow demonstrates how to discover records from the network: - -1. Search for records by skill - - ```bash - # Use JSON output to process results - dirctl routing search --skill "AI" --limit 10 --output json - ``` - -1. Search with multiple criteria - - ```bash - dirctl routing search --skill "AI" --locator "docker-image" --min-score 2 --output json - ``` - -1. Pull discovered records - ```bash - # Extract CIDs and pull records - dirctl routing search --skill "AI" --output json | \ - jq -r '.[].record_ref.cid' | \ - xargs -I {} dirctl pull {} - ``` - -### Synchronization Workflow - -The following workflow demonstrates how to synchronize records between remote directories and your local instance: - -1. Create sync with remote peer - - ```bash - # Using --output raw for clean variable capture - SYNC_ID=$(dirctl sync create https://peer.example.com --output raw) - echo "Sync created with ID: $SYNC_ID" - ``` - -1. Monitor sync progress - - ```bash - # Use JSON output for programmatic monitoring - dirctl sync status $SYNC_ID --output json - ``` - -1. List all syncs - - ```bash - # Use JSONL output for streaming results - dirctl sync list --output jsonl - ``` - -1. Clean up when done - - ```bash - dirctl sync delete $SYNC_ID - ``` - -### Advanced Synchronization: Search-to-Sync Pipeline - -Automatically sync records that match specific criteria from the network: - -```bash -# Search for AI-related records and create sync operations -dirctl routing search --skill "AI" --output json | dirctl sync create --stdin - -# This creates separate sync operations for each remote peer found, -# syncing only the specific CIDs that matched your search criteria -``` - -### Import Workflow - -Import records from external registries (e.g. MCP registry) with optional signing and rate limiting: - -```bash -# 1. Preview import with dry run -dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --limit=10 \ - --dry-run - -# 2. Perform actual import with debug output -dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --filter=updated_since=2025-08-07T13:15:04.280Z \ - --debug - -# 3. Force reimport to update existing records -dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --limit=10 \ - --force - -# 4. Import with signing enabled -dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --limit=5 \ - --sign - -# 5. Import with rate limiting for LLM API calls -dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --enrich-rate-limit=5 \ - --debug - -# 6. Search imported records -dirctl search --query "module=runtime/mcp" -``` - -### Event Streaming Workflow - -Listen to directory events and process them (e.g. filter by type or labels): - -```bash -# Listen to all events (human-readable) -dirctl events listen - -# Stream events as JSONL for processing -dirctl events listen --output jsonl | jq -c . - -# Filter and process specific event types -dirctl events listen --types RECORD_PUSHED --output jsonl | \ - jq -c 'select(.type == "EVENT_TYPE_RECORD_PUSHED")' | \ - while read event; do - CID=$(echo "$event" | jq -r '.resource_id') - echo "New record pushed: $CID" - done - -# Monitor events with label filters -dirctl events listen --labels /skills/AI --output jsonl | \ - jq -c '.resource_id' >> ai-records.log - -# Extract just resource IDs from events -dirctl events listen --output raw | tee event-cids.txt -``` - -## Output Formats - -All `dirctl` commands support multiple output formats via the `--output` (or `-o`) flag, making it easy to switch between human-readable output and machine-processable formats. - -### Available Formats - -| Format | Description | Use Case | -|--------|-------------|----------| -| `human` | Human-readable, formatted output with colors and tables (default) | Interactive terminal use | -| `json` | Pretty-printed JSON with indentation | Debugging, single-record processing | -| `jsonl` | Newline-delimited JSON (compact, one object per line) | Streaming, batch processing, logging | -| `raw` | Raw values only (e.g., CIDs, IDs) | Shell scripting, piping to other commands | - -### Usage - -```bash -# Human-readable output (default) -dirctl routing list - -# JSON output (pretty-printed) -dirctl routing list --output json -dirctl routing list -o json - -# JSONL output (streaming-friendly) -dirctl events listen --output jsonl - -# Raw output (just values) -dirctl push my-agent.json --output raw -``` - -### Piping and Processing - -Structured formats (`json`, `jsonl`, `raw`) automatically route data to **stdout** and metadata messages to **stderr**, enabling clean piping to tools like `jq`: - -```bash -# Process JSON output with jq -dirctl routing search --skill "AI" -o json | jq '.[] | .cid' - -# Stream events and filter by type -dirctl events listen -o jsonl | jq -c 'select(.type == "EVENT_TYPE_RECORD_PUSHED")' - -# Capture CID for scripting -CID=$(dirctl push my-agent.json -o raw) -echo "Stored with CID: $CID" - -# Chain commands -dirctl routing list -o json | jq -r '.[].cid' | xargs -I {} dirctl pull {} -``` - -### Format Selection Guidelines - -- **`human`**: Default for terminal interaction, provides context and formatting -- **`json`**: Best for debugging or when you need readable structured data -- **`jsonl`**: Ideal for streaming events, logs, or processing large result sets line-by-line -- **`raw`**: Perfect for shell scripts and command chaining where you only need the value - -## Authentication - -Authentication is required when accessing Directory federation nodes. The CLI supports multiple authentication modes, with GitHub OAuth recommended for interactive use. - -| Command | Description | -|---------|-------------| -| `dirctl auth login` | Authenticate with GitHub | -| `dirctl auth logout` | Clear cached authentication credentials | -| `dirctl auth status` | Show current authentication status | - -### GitHub OAuth Authentication - -GitHub OAuth (Device Flow) enables secure, interactive authentication for accessing federation nodes. - -#### `dirctl auth login` - -Authenticate with GitHub using the OAuth 2.0 Device Flow. No prerequisites. - -```bash -# Start login (shows a code and link) -dirctl auth login -``` - -What happens: - -1. The CLI displays a short-lived **code** (e.g. `9190-173C`) and the URL **https://github.com/login/device** -2. You open that URL (on this machine or any device), enter the code, and authorize the application -3. After you authorize, the CLI receives a token and caches it at `~/.config/dirctl/auth-token.json` -4. Subsequent commands automatically use the cached token (no `--auth-mode` flag needed) - -```bash -# Force re-login even if already authenticated -dirctl auth login --force - -# Show code and URL only (do not open browser automatically) -dirctl auth login --no-browser -``` - -!!! note "Custom OAuth App (optional)" - To use your own GitHub OAuth App instead of the default, create an OAuth App in [GitHub Developer Settings](https://github.com/settings/developers) with Device Flow support and set `DIRECTORY_CLIENT_GITHUB_CLIENT_ID` (and optionally `DIRECTORY_CLIENT_GITHUB_CLIENT_SECRET`). For normal use, leave these unset. - -#### `dirctl auth status` - -Check your current authentication status. - -```bash -# Show authentication status -dirctl auth status - -# Validate token with GitHub API -dirctl auth status --validate -``` - -Example output: - -``` -Status: Authenticated - User: your-username - Organizations: agntcy, your-org - Cached at: 2025-12-22T10:30:00Z - Token: Valid ✓ - Estimated expiry: 2025-12-22T18:30:00Z - Cache file: /Users/you/.config/dirctl/auth-token.json -``` - -#### `dirctl auth logout` - -Clear cached authentication credentials. - -```bash -dirctl auth logout -``` - -#### Using Authenticated Commands - -Once authenticated via `dirctl auth login`, your cached credentials are automatically detected and used: - -```bash -# Push to federation (auto-detects and uses cached GitHub credentials) -dirctl push my-agent.json - -# Search federation nodes (auto-detects authentication) -dirctl --server-addr=federation.agntcy.org:443 search --skill "natural_language_processing" - -# Pull from federation (auto-detects authentication) -dirctl pull baeareihdr6t7s6sr2q4zo456sza66eewqc7huzatyfgvoupaqyjw23ilvi -``` - -**Authentication mode behavior:** - -- **No `--auth-mode` flag (default)**: Auto-detects authentication in this order: SPIFFE (if available in Kubernetes/SPIRE environment), cached GitHub credentials (if `dirctl auth login` was run), then insecure (for local development). -- **Explicit `--auth-mode=github`**: Forces GitHub authentication (e.g. to bypass SPIFFE in a SPIRE environment). -- **Other modes**: Use `--auth-mode=x509`, `--auth-mode=jwt`, or `--auth-mode=tls` for specific authentication methods. - -```bash -# Force GitHub auth even if SPIFFE is available -dirctl --auth-mode=github push my-agent.json -``` - -### Other Authentication Modes - -| Mode | Description | Use Case | -|------|-------------|----------| -| `github` | GitHub OAuth (explicit) | Force GitHub auth, bypass SPIFFE auto-detect | -| `x509` | SPIFFE X.509 certificates | Kubernetes workloads with SPIRE | -| `jwt` | SPIFFE JWT tokens | Service-to-service authentication | -| `token` | SPIFFE token file | Pre-provisioned credentials | -| `tls` | mTLS with certificates | Custom PKI environments | -| `insecure` / `none` | No auth, skip auto-detect | Testing, local development | -| (empty) | Auto-detect: SPIFFE → cached GitHub → insecure | Default behavior (recommended) | - -## Configuration - -### Server Connection - -```bash -# Connect to specific server -dirctl --server-addr localhost:8888 routing list - -# Use environment variable -export DIRECTORY_CLIENT_SERVER_ADDRESS=localhost:8888 -dirctl routing list -``` - -### Authentication - -```bash -# Use SPIFFE Workload API -dirctl --spiffe-socket-path /run/spire/sockets/agent.sock routing list -``` - -## Command Organization - -The CLI follows a clear service-based organization: - -- **Auth**: GitHub OAuth authentication (`auth login`, `auth logout`, `auth status`). -- **Storage**: Direct record management (`push`, `pull`, `delete`, `info`). -- **Import**: Batch imports from external registries (`import`). -- **Routing**: Network announcement and discovery (`routing publish`, `routing list`, `routing search`). -- **Search**: General content search (`search`). -- **Security**: Signing, verification, and validation (`sign`, `verify`, `validate`, `naming verify`). -- **Sync**: Peer synchronization (`sync`). - -Each command group provides focused functionality with consistent flag patterns and clear separation of concerns. - -## Getting Help - -Use the following commands to get help with the CLI: - -- General help - ```bash - dirctl --help - ``` - -- Command group help - ```bash - dirctl routing --help - ``` - -- Specific command help - ```bash - dirctl routing search --help - ``` - -For more advanced usage, troubleshooting, and development workflows, see the [AGNTCY documentation](https://docs.agntcy.org/dir/overview/). diff --git a/docs/dir/directory-gui.md b/docs/dir/directory-gui.md deleted file mode 100644 index 26a6dca..0000000 --- a/docs/dir/directory-gui.md +++ /dev/null @@ -1,118 +0,0 @@ -# Agent Directory GUI - -Welcome to the **AGNTCY Directory GUI**. This application provides a visual -interface for interacting with the Model Context Protocol (MCP) server, offering -a streamlined experience for managing directory resources through natural -language. - -## Architecture: A Unified Bundle - -Typically, the Model Context Protocol involves a client (like an IDE or a chat -bot) talking to a separate server process. Configuring this requires managing -paths, ports, and binaries manually. - -**The Directory GUI simplifies this:** - -- **Embedded Server**: The application embeds the MCP Server (written in Go) -directly within the app bundle. -- **Automatic Lifecycle**: When you launch the GUI, it automatically starts the -MCP server in the background and establishes a secure connection. -- **Self-Contained**: You don't need to install or run a separate server -command; the app handles everything. - -## How it Works - -Using the GUI is similar to using an AI coding assistant (like GitHub Copilot -Chat), but with a specialized focus: - -1. **Chat Interface**: You type commands in natural language (e.g., "List all -agents," "Register a new service," "Find events related to X"). -2. **Tool Execution**: The LLM interprets your request and decides to call -specific **MCP Tools**. -3. **Visualization**: instead of just seeing text or JSON, the GUI can render -specialized views for the data returned by the tools, providing a richer -experience than a standard terminal. - -It bridges the gap between raw CLI commands and high-level intent, allowing you -to "chat" with your system architecture. - -## Settings & Configuration - -The **Settings Screen** allows you to configure the "Brain" of the application -(the LLM) and the connections to external services. - -### LLM Providers -You can switch between different AI models to power the chat experience: - -* **Google Gemini**: Use Google's generative AI models. -* **Azure OpenAI**: Connect to your enterprise Azure deployments. -* **OpenAI Compatible**: Connect to standard OpenAI APIs or compatible proxies. -* **Ollama (Local)**: Use local models running on your machine (e.g., `gemma3:4b`, `llama3`). This enables a fully local stack where data never leaves your device. - -### Configuration Fields -* **API Keys**: Securely input keys for cloud providers. -* **Endpoints**: Custom endpoints for Azure or local servers. -* **Model Selection**: Specify exactly which model version you want to use (e.g., defaulting to `gemma3:4b` for Ollama). - -### Directory Connection -* **Server Address**: Point the internal MCP server to your remote or local Directory Service instance. -* **Authentication**: detailed configuration for tokens to ensure secure access to your directory data. - - - - -## Downloads - -You can find the latest build artifacts for each platform below: - -* **Windows**: [Download Windows App](https://github.com/agntcy/dir/releases/download/gui/v1.0.0/agntcy-directory-windows.zip) -* **macOS**: [Download macOS App](https://github.com/agntcy/dir/releases/download/gui/v1.0.0/AGNTCY-Directory.dmg) - -or check the [Release Page](https://github.com/agntcy/dir/releases/tag/gui%2Fv1.0.0). - -## Source Code - -The source code for the GUI is available in the `gui` folder of the [dir -repository](https://github.com/agntcy/dir/tree/main/gui). - -Build and release workflows are defined in -[gui-ci.yaml](https://github.com/agntcy/dir/blob/main/.github/workflows/gui-ci.yaml). - -## macOS Troubleshooting - -macOS apps without an Apple Developer Program attestation (code signing/notarization) -can be built using Xcode, but they will be blocked by Gatekeeper upon installation, -displaying a malware warning. Users can bypass this by overriding **Privacy & Security** -settings in **System Settings** to "Open Anyway". - -Alternatively, you can run the following command in your terminal to clear the quarantine attribute: - -```bash -xattr -d com.apple.quarantine /Applications/AGNTCY\ Directory.app -``` - -## Windows Troubleshooting - -The "Windows protected your PC" message (Microsoft Defender SmartScreen) -commonly appears when installing software downloaded from GitHub because the -application is unrecognized, unsigned, or lacks a high reputation score. - -### How to Run the App ("Run Anyway") - -To unblock an .exe file in Windows 11, right-click the file, select Properties, -check the Unblock box under the "General" tab, and click Apply. If the option is -missing, the file is not blocked. - -You can also bypass SmartScreen by clicking "More info" > "Run anyway". - -### Why This Happens -* **Unknown Publisher/Unsigned Code**: Many independent developers on GitHub - do not purchase expensive digital code-signing certificates. Windows - defaults to blocking these. -* **Low Reputation**: Even if signed, new apps need time for Windows to build - a "reputation score" based on user adoption. New or rarely downloaded tools - will trigger this warning. -* **False Positives**: Sometimes, generic malware detection flags legitimate - software, particularly in installer packages. - - diff --git a/docs/dir/directory-mcp.md b/docs/dir/directory-mcp.md deleted file mode 100644 index 04cc71d..0000000 --- a/docs/dir/directory-mcp.md +++ /dev/null @@ -1,108 +0,0 @@ -# Directory MCP Server - -The Directory MCP Server provides a standardized interface for AI assistants and tools to interact with the AGNTCY Agent Directory and work with OASF agent records. - -The Directory MCP Server exposes Directory functionality through MCP, allowing AI assistants to: - -- Work with OASF schemas and validate agent records. -- Search and discover agent records from the Directory. -- Push and pull records to/from Directory servers. -- Navigate OASF skill and domain taxonomies. -- Generate agent records automatically from codebases. - -The MCP server runs via the `dirctl` CLI tool and acts as a bridge between AI development environments and the Directory infrastructure, making it easier to work with agent metadata in your development workflow. - -## Configuration - -### Binary Configuration - -Add the MCP server to your IDE's MCP configuration using the absolute path to the `dirctl` binary. - -The server requires `OASF_API_VALIDATION_SCHEMA_URL` (see [Environment variables](#environment-variables)); without it the process exits on startup. - -**Example Cursor configuration (`~/.cursor/mcp.json`):** - -```json -{ - "mcpServers": { - "dir-mcp-server": { - "command": "/absolute/path/to/dirctl", - "args": ["mcp", "serve"], - "env": { - "OASF_API_VALIDATION_SCHEMA_URL": "https://schema.oasf.outshift.com" - } - } - } -} -``` - -### Docker Configuration - -Add the MCP server to your IDE's MCP configuration using Docker. Pass the same schema URL via `--env` (the container does not set a default). - -??? example "Example Cursor configuration (`~/.cursor/mcp.json`)" - - ```json - { - "mcpServers": { - "dir-mcp-server": { - "command": "docker", - "args": [ - "run", - "--rm", - "-i", - "--env", - "OASF_API_VALIDATION_SCHEMA_URL=https://schema.oasf.outshift.com", - "ghcr.io/agntcy/dir-ctl:latest", - "mcp", - "serve" - ] - } - } - } - ``` - -### Environment Variables - -Configure the MCP server behavior using environment variables: - -- `OASF_API_VALIDATION_SCHEMA_URL` - Required. Base URL of the OASF schema API used to load schemas for validation (for example `https://schema.oasf.outshift.com`). The server will not start if this is unset. -- `DIRECTORY_CLIENT_SERVER_ADDRESS` - Directory server address (default: `0.0.0.0:8888`) -- `DIRECTORY_CLIENT_AUTH_MODE` - Authentication mode: `none`, `x509`, `jwt`, `token` -- `DIRECTORY_CLIENT_SPIFFE_TOKEN` - Path to SPIFFE token file (for token authentication) -- `DIRECTORY_CLIENT_TLS_SKIP_VERIFY` - Skip TLS verification (set to `true` if needed) - -??? example "Example with environment variables" - - ```json - { - "mcpServers": { - "dir-mcp-server": { - "command": "/usr/local/bin/dirctl", - "args": ["mcp", "serve"], - "env": { - "DIRECTORY_CLIENT_SERVER_ADDRESS": "dir.example.com:8888", - "DIRECTORY_CLIENT_AUTH_MODE": "none", - "DIRECTORY_CLIENT_TLS_SKIP_VERIFY": "false" - } - } - } - } - ``` - -## Directory MCP Server Tools - -Using the Directory MCP Server, you can access the following tools: - -- `agntcy_oasf_list_versions` - Lists all available OASF schema versions supported by the server. -- `agntcy_oasf_get_schema` - Retrieves the complete OASF schema JSON content for the specified version. -- `agntcy_oasf_get_schema_skills` - Retrieves skills from the OASF schema with hierarchical navigation support. -- `agntcy_oasf_get_schema_domains` - Retrieves domains from the OASF schema with hierarchical navigation support. -- `agntcy_oasf_validate_record` - Validates an OASF agent record against the OASF schema. -- `agntcy_dir_push_record` - Pushes an OASF agent record to a Directory server. -- `agntcy_dir_pull_record` - Pulls an OASF agent record from the local Directory node by its CID (Content Identifier). -- `agntcy_dir_search_local` - Searches for agent records on the local directory node using structured query filters. -- `agntcy_dir_verify_record` - Verifies the digital signature of a record in the Directory by its CID (server-side integrity and authenticity). -- `agntcy_dir_verify_name` - Verifies that a record's name is owned by the domain it claims (by CID or name; optional version). Checks that the record was signed with a key published in the domain's well-known JWKS file. - -For a full list of tools and usage examples, see the [Directory MCP Server documentation](https://github.com/agntcy/dir/blob/main/mcp/README.md). diff --git a/docs/dir/directory-sdk.md b/docs/dir/directory-sdk.md deleted file mode 100644 index e9eccd9..0000000 --- a/docs/dir/directory-sdk.md +++ /dev/null @@ -1,146 +0,0 @@ -# Directory SDK - -The Directory SDK provides comprehensive libraries and tools for interacting with the Directory system, including storage, routing, search, and security operations. - -## JavaScript SDK - -Documentation for the JavaScript SDK can be found on [GitHub](https://github.com/agntcy/dir/tree/main/sdk/dir-js). The SDK supports both JavaScript and TypeScript applications. The package is published on [npm](https://www.npmjs.com/package/agntcy-dir). - -!!! note - - The SDK is intended for use in Node.js applications and does not work in web applications. - -### Installation - -Install the SDK using one of available JS package managers like [npm](https://www.npmjs.com/) - -1. Initialize the project: - - ```bash - npm init -y - ``` - -1. Add the SDK to your project: - - ```bash - npm install agntcy-dir - ``` - -### Configuration - -The SDK can be configured via environment variables or direct instantiation: - -```js -import {Config, Client} from 'agntcy-dir'; - -// Environment variables -process.env.DIRECTORY_CLIENT_SERVER_ADDRESS = "localhost:8888"; -process.env.DIRCTL_PATH = "/path/to/dirctl"; -const client = new Client(); - -// Or configure directly -const config = new Config( - serverAddress="localhost:8888", - dirctlPath="/usr/local/bin/dirctl" -); -const client = new Client(config); - -// Use SPIRE for mTLS communication -const config = new Config(spiffeEndpointSocket="/tmp/agent.sock"); -const transport = await Client.createGRPCTransport(config); -const spiffeClient = new Client(config, transport); -``` - -!!! note - - JavaScript SDK requires Directory CLI (dirctl) only to perform signing operations. If you don't need signing, you can use the SDK without dirctl. - -## Python SDK - -Documentation for the Python SDK can be found on [GitHub](https://github.com/agntcy/dir/tree/main/sdk/dir-py). -The SDK supports Python 3.10+ applications. The package is published on [PyPI](https://pypi.org/project/agntcy-dir/). - -### Installation - -Install the SDK using [uv](https://github.com/astral-sh/uv) - -1. Initialize the project: - ```bash - uv init - ``` - -1. Add the SDK to your project: - ```bash - uv add agntcy-dir --index https://buf.build/gen/python - ``` - -### Configuration - -The SDK can be configured via environment variables or direct instantiation: - -```python -from agntcy.dir_sdk.client import Config, Client - -# Environment variables -export DIRECTORY_CLIENT_SERVER_ADDRESS="localhost:8888" -export DIRCTL_PATH="/path/to/dirctl" -client = Client() - -# Or configure directly -config = Config( - server_address="localhost:8888", - dirctl_path="/usr/local/bin/dirctl" -) -client = Client(config) - -# Use SPIRE for mTLS communication -config = Config( - spiffe_socket_path="/tmp/agent.sock" -) -client = Client(config) -``` - -!!! note - - Python SDK requires Directory CLI (dirctl) only to perform signing operations. If you don't need signing, you can use the SDK without dirctl. - -## Go SDK - -Documentation for the Go SDK can be found at [GitHub](https://github.com/agntcy/dir/tree/main/client). The package is available on [pkg.go.dev](https://pkg.go.dev/github.com/agntcy/dir/client). - -### Installation - -Install the SDK using `go get` - -```bash -go get github.com/agntcy/dir/client -``` - -### Configuration - -The SDK can be configured via environment variables or direct instantiation: - -```go -import "github.com/agntcy/dir/client" - -// Environment variables -os.Setenv("DIRECTORY_CLIENT_SERVER_ADDRESS", "localhost:8888") -os.Setenv("DIRCTL_PATH", "/path/to/dirctl") -client := client.New() - -// Or configure directly -config := &client.Config{ - ServerAddress: "localhost:8888", -} -client := client.New(client.WithConfig(config)) - -// Use SPIRE for mTLS communication -config := &client.Config{ - SpiffeSocketPath: "/tmp/agent.sock", -} -client := client.New(client.WithConfig(config)) -``` - -!!! note - - Golang SDK does not require Directory CLI (dirctl) and can be used standalone. diff --git a/docs/dir/events.md b/docs/dir/events.md deleted file mode 100644 index 9a22776..0000000 --- a/docs/dir/events.md +++ /dev/null @@ -1,505 +0,0 @@ -# Event Streaming - -The Directory Event Service provides real-time monitoring of system operations through a -lightweight streaming API. This enables external applications, monitoring systems, and automation -tools to react to changes as they occur across the distributed directory network. - -## Overview - -Events are emitted for key operations across all Directory services, including storage, routing, -synchronization, and signing activities. The event system delivers notifications in real-time from -the moment of subscription, without maintaining history or supporting replay functionality. - -### Supported Event Types - -The Directory tracks 9 distinct event types organized by service: - -| Service | Event Types | Description | -|---------|-------------|-------------| -| **Store** | RECORD_PUSHED
RECORD_PULLED
RECORD_DELETED | Local storage operations | -| **Routing** | RECORD_PUBLISHED
RECORD_UNPUBLISHED | Network announcement operations | -| **Sync** | SYNC_CREATED
SYNC_COMPLETED
SYNC_FAILED | Directory synchronization operations | -| **Sign** | RECORD_SIGNED | Cryptographic signing operations | - -## Key Characteristics - -- **Real-time delivery**: Events are delivered immediately as operations occur. -- **Forward-only**: No historical events or replay capability. -- **Filtered subscriptions**: Server-side filtering by event type, labels, and CIDs. -- **Type-safe**: Protocol buffer enums prevent invalid event types. -- **Asynchronous**: Event publishing never blocks API operations. - -## Using the CLI - -The `dirctl events listen` command provides real-time event monitoring with flexible filtering -options. - -### Listen to All Events - -```bash -# Listen to all system events -dirctl events listen -``` - -**Output:** - -``` -Listening to events (press Ctrl+C to stop)... - -[14:23:15] RECORD_PUSHED: bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi -[14:23:17] RECORD_PUBLISHED: bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi -[14:23:20] SYNC_CREATED: sync-12345 -[14:23:35] SYNC_COMPLETED: sync-12345 {"remote_url":"https://remote.dir.example.com","record_count":"5"} -``` - -### Filter by Event Type - -```bash -# Only receive storage events -dirctl events listen --types RECORD_PUSHED,RECORD_PULLED,RECORD_DELETED - -# Only receive routing events -dirctl events listen --types RECORD_PUBLISHED,RECORD_UNPUBLISHED - -# Only receive synchronization events -dirctl events listen --types SYNC_CREATED,SYNC_COMPLETED,SYNC_FAILED -``` - -### Filter by Labels - -Monitor events for records with specific skill taxonomies or other labels: - -```bash -# Monitor AI-related records -dirctl events listen --labels /skills/AI - -# Monitor specific research domain -dirctl events listen --labels /domains/research - -# Combine multiple labels -dirctl events listen --labels /skills/AI,/domains/research -``` - -### Filter by CID - -Track operations on specific records: - -```bash -# Monitor a specific record -dirctl events listen --cids bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi - -# Monitor multiple records -dirctl events listen --cids bafybe...,bafkyb... -``` - -### JSON Output - -For programmatic processing: - -```bash -# JSONL format output (streaming-friendly, one event per line) -dirctl events listen --output jsonl - -# JSON format output (pretty-printed) -dirctl events listen --output json -``` - -**JSONL format** (recommended for streaming - compact, one event per line): -```json -{"id":"550e8400-...","type":"EVENT_TYPE_RECORD_PUSHED","timestamp":"2025-10-18T14:23:15.123456Z","resource_id":"bafybeig...","labels":["/skills/AI"]} -{"id":"550e8401-...","type":"EVENT_TYPE_RECORD_PUBLISHED","timestamp":"2025-10-18T14:23:16.456789Z","resource_id":"bafybeig...","labels":["/skills/AI"]} -``` - -**JSON format** (pretty-printed): -```json -{ - "id": "550e8400-e29b-41d4-a716-446655440000", - "type": "EVENT_TYPE_RECORD_PUSHED", - "timestamp": "2025-10-18T14:23:15.123456Z", - "resource_id": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", - "labels": ["/skills/AI", "/domains/research"] -} -``` - -> **Note:** For streaming events, use `--output jsonl` format as it outputs one compact JSON object per line, making it ideal for real-time processing with tools like `jq`. - -### Combine Filters - -All filter types can be combined for precise event monitoring: - -```bash -# Monitor AI record storage operations -dirctl events listen \ - --types RECORD_PUSHED,RECORD_PULLED \ - --labels /skills/AI \ - --output jsonl -``` - -## Using the Go SDK - -The Directory Go client provides a streaming API for event consumption: - -```go -package main - -import ( - "context" - "fmt" - - "github.com/agntcy/dir/client" - eventsv1 "github.com/agntcy/dir/api/events/v1" -) - -func main() { - // Create client - c, err := client.New( - client.WithAddress("localhost:8080"), - ) - if err != nil { - panic(err) - } - defer c.Close() - - // Subscribe to all events - ctx := context.Background() - result, err := c.ListenStream(ctx, &eventsv1.ListenRequest{}) - if err != nil { - panic(err) - } - - // Receive events using StreamResult pattern - for { - select { - case resp := <-result.ResCh(): - event := resp.GetEvent() - fmt.Printf("[%s] %s: %s\n", - event.GetTimestamp().AsTime().Format("15:04:05"), - event.GetType(), - event.GetResourceId()) - case err := <-result.ErrCh(): - panic(err) - case <-result.DoneCh(): - return - case <-ctx.Done(): - return - } - } -} -``` - -### Subscribe with Filters with Go SDK - -```go -// Subscribe to specific event types -result, err := c.ListenStream(ctx, &eventsv1.ListenRequest{ - EventTypes: []eventsv1.EventType{ - eventsv1.EventType_EVENT_TYPE_RECORD_PUSHED, - eventsv1.EventType_EVENT_TYPE_RECORD_PUBLISHED, - }, -}) - -// Subscribe with label filter -result, err := c.ListenStream(ctx, &eventsv1.ListenRequest{ - LabelFilters: []string{"/skills/AI"}, -}) - -// Subscribe with CID filter -result, err := c.ListenStream(ctx, &eventsv1.ListenRequest{ - CidFilters: []string{"bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"}, -}) -``` - -## Using the Python SDK - -```python -from agntcy.dir_sdk.client import Client -from agntcy.dir_sdk.models import events_v1 - -# Create client -client = Client(address="localhost:8080") - -# Subscribe to events -request = events_v1.ListenRequest() -stream = client.events_client.Listen(request) - -# Receive events -try: - for response in stream: - event = response.event - print(f"[{event.type}] {event.resource_id}") -except KeyboardInterrupt: - pass -finally: - client.close() -``` - -### Subscribe with Filters with Python SDK - -```python -# Subscribe to specific event types -request = events_v1.ListenRequest( - event_types=[ - events_v1.EventType.EVENT_TYPE_RECORD_PUSHED, - events_v1.EventType.EVENT_TYPE_RECORD_PUBLISHED, - ] -) - -# Subscribe with label filter -request = events_v1.ListenRequest( - label_filters=["/skills/AI"] -) - -# Subscribe with CID filter -request = events_v1.ListenRequest( - cid_filters=["bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"] -) -``` - -## Use Cases - -### Monitoring and Observability - -Track system activity for operational awareness: - -```bash -# Monitor all operations in production -dirctl events listen --output jsonl | tee events.log - -# Alert on failed syncs -dirctl events listen --types SYNC_FAILED | \ - while read line; do - echo "ALERT: $line" - # Send notification - done -``` - -### Automation and Integration - -React to events programmatically: - -```go -// Automatically publish records after they're pushed -for { - select { - case resp := <-result.ResCh(): - event := resp.GetEvent() - if event.GetType() == eventsv1.EventType_EVENT_TYPE_RECORD_PUSHED { - cid := event.GetResourceId() - // Automatically publish to network - _, err := c.Publish(ctx, &routingv1.PublishRequest{ - RecordRef: &corev1.RecordRef{Cid: cid}, - }) - } - case err := <-result.ErrCh(): - return fmt.Errorf("stream error: %w", err) - case <-result.DoneCh(): - return nil - case <-ctx.Done(): - return ctx.Err() - } -} -``` - -### Audit and Compliance - -Maintain audit trails for critical operations: - -```bash -# Record all signature events -dirctl events listen --types RECORD_SIGNED --output jsonl >> audit.jsonl -``` - -### Development and Debugging - -Monitor system behavior during development: - -```bash -# Watch all events during testing -dirctl events listen --output jsonl | jq -c . - -# Track specific record through the system -dirctl events listen --cids $RECORD_CID -``` - -## Event Lifecycle - -Events are emitted at specific points in operation lifecycles: - -### Storage Operations - -```mermaid -sequenceDiagram - participant Client - participant Store - participant EventBus - participant Subscribers - - Client->>Store: Push record - Store->>Store: Validate & store - Store->>EventBus: Emit RECORD_PUSHED - EventBus->>Subscribers: Deliver event - Store->>Client: Return CID -``` - -### Synchronization Operations - -```mermaid -sequenceDiagram - participant Client - participant Sync - participant EventBus - participant Subscribers - - Client->>Sync: Create sync - Sync->>EventBus: Emit SYNC_CREATED - EventBus->>Subscribers: Deliver event - Sync->>Sync: Execute sync - alt Success - Sync->>EventBus: Emit SYNC_COMPLETED - else Failure - Sync->>EventBus: Emit SYNC_FAILED - end - EventBus->>Subscribers: Deliver event -``` - -## Best Practices - -### For Client Applications - -#### Handle Connection Failures - -Implement retry logic with exponential backoff: - -```go -backoff := time.Second -for { - result, err := c.ListenStream(ctx, req) - if err != nil { - time.Sleep(backoff) - backoff *= 2 - continue - } - - // Process events... - for { - select { - case resp := <-result.ResCh(): - // Process event - case err := <-result.ErrCh(): - // Connection error - will retry with backoff - goto reconnect - case <-result.DoneCh(): - goto reconnect - case <-ctx.Done(): - return - } - } - -reconnect: - time.Sleep(backoff) - backoff = min(backoff*2, time.Minute) -} -``` - -#### Use Appropriate Filters - -Filter server-side to reduce bandwidth and processing: - -```go -// ✅ Good: Filter on server -result, _ := c.ListenStream(ctx, &eventsv1.ListenRequest{ - EventTypes: []eventsv1.EventType{eventsv1.EventType_EVENT_TYPE_RECORD_PUSHED}, -}) - -// ❌ Bad: Filter on client -result, _ := c.ListenStream(ctx, &eventsv1.ListenRequest{}) -for { - select { - case resp := <-result.ResCh(): - if resp.GetEvent().GetType() == eventsv1.EventType_EVENT_TYPE_RECORD_PUSHED { - // Process... - } - // ... other cases - } -} -``` - -#### Set Appropriate Timeouts - -Use context timeouts to prevent hanging connections: - -```go -ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) -defer cancel() - -result, err := c.ListenStream(ctx, req) -``` - -### For Production Deployments - -- **Monitor dropped events**: Check server logs for slow consumer warnings -- **Use structured logging**: JSON output enables log aggregation and analysis -- **Implement graceful shutdown**: Properly close streams on application termination -- **Consider buffering**: Local buffering can smooth over temporary processing delays - -## Limitations - -Understanding the event system's design constraints: - -### No Event History - -Events are only delivered from subscription time forward. Historical events are not stored or -available for replay. Applications requiring event history must maintain their own storage. - -### At-Most-Once Delivery - -If a subscriber cannot keep up with event delivery, events may be dropped. The system prioritizes -overall performance over guaranteed delivery to slow consumers. - -### No Ordering Guarantees - -Events from different services may arrive in any order. Applications requiring strict ordering -should use timestamps or implement their own sequencing logic. - -### Per-Server Scope - -Each Directory server maintains its own event bus. Events are not propagated across server -instances in a distributed deployment. - -## Configuration - -Event system behavior can be tuned via server configuration: - -```yaml -# /etc/agntcy/dir/server.config.yml -events: - # Buffer size per subscriber (default: 100) - # Larger values allow subscribers to fall behind temporarily - subscriber_buffer_size: 100 - - # Log warnings when events are dropped (default: true) - log_slow_consumers: true - - # Debug logging for all events (default: false) - # Very verbose - use only for debugging - log_published_events: false -``` - -**Environment Variables:** - -```bash -DIRECTORY_SERVER_EVENTS_SUBSCRIBER_BUFFER_SIZE=100 -DIRECTORY_SERVER_EVENTS_LOG_SLOW_CONSUMERS=true -DIRECTORY_SERVER_EVENTS_LOG_PUBLISHED_EVENTS=false -``` - -!!! note - The `subscriber_buffer_size` parameter controls memory usage. Larger buffers reduce event - dropping but consume more memory per subscriber. - -## See Also - -- [Events API Reference](https://buf.build/agntcy/dir/docs/main:agntcy.dir.events.v1) - Protocol buffer definitions and message formats -- [CLI Reference](directory-cli.md) - Complete CLI command documentation -- [SDK Reference](directory-sdk.md) - Language-specific SDK usage guides -- [Getting Started](getting-started.md) - Initial setup and deployment guide diff --git a/docs/dir/federation-profiles.md b/docs/dir/federation-profiles.md deleted file mode 100644 index f8a2d2b..0000000 --- a/docs/dir/federation-profiles.md +++ /dev/null @@ -1,361 +0,0 @@ -# Federation Bundle Profiles - -Directory supports two federation bundle profiles for secure trust bundle exchange between SPIRE servers. Each profile implements distinct security models, infrastructure requirements, and operational characteristics. This document provides technical guidance to assist in profile selection and implementation. - -For step-by-step federation setup with the public Directory network, see [Running a Federated Directory Instance](partner-prod-federation.md). - -## Profile Comparison - -| Aspect | https_web | https_spiffe | -|--------|-----------|--------------| -| Transport Protocol | Standard HTTPS | SPIFFE mutual TLS | -| Certificate Type | CA-signed X.509 (Let's Encrypt, Enterprise CA) | SPIFFE X.509-SVID | -| Certificate Validation | DNS SANs with CA chain verification | SPIFFE ID URI SAN with Trust Bundle | -| SSL Passthrough | Not required | Required | -| Bootstrap Bundle | Not required | Required | -| cert-manager Dependency | Required | Not required | -| DNS Requirement | Public DNS resolution required | Optional (IP addresses supported) | -| Traffic Inspection | Supported (NGINX terminates TLS) | Not supported (end-to-end encryption) | -| CA Trust Model | Relies on public or enterprise CA | Self-contained SPIFFE trust | -| Implementation Complexity | Medium | Medium-High | -| Estimated Setup Time | Approximately 30 minutes | Approximately 45 minutes | - -The profiles are described in more detail below: - -=== "https_web profile" - - Federation over standard HTTPS using CA-signed certificates. The SPIRE server presents a certificate validated through traditional DNS Subject Alternative Names (SANs) and CA certificate chain verification. This profile leverages existing PKI infrastructure and standard TLS implementations. - - **Architecture:** - - ```mermaid - flowchart TB - Remote[Remote SPIRE Bundle Consumer] - NGINX[NGINX Ingress Controller
• Terminates TLS Let's Encrypt
• Traffic inspection/logging
• Initiates TLS to backend] - SPIRE[SPIRE Server Bundle Provider] - - Remote -->|"Standard HTTPS (validates DNS SAN)"| NGINX - NGINX -->|"Internal HTTPS with SNI"| SPIRE - ``` - - **Advantages:** - - 1. **No SSL Passthrough Requirement** - - - Compatible with standard NGINX configurations - - Aligns with enterprise security policies restricting SSL passthrough - - No special ingress controller configuration required - - 2. **Simplified Initial Trust Establishment** - - - Remote SPIRE server can fetch bundles immediately - - No manual trust bundle exchange required - - Eliminates out-of-band coordination overhead - - 3. **Traffic Inspection Compatibility** - - - Web Application Firewalls (WAF) can inspect traffic - - Load balancers maintain request/response visibility - - DDoS protection operates at Layer 7 (application layer) - - 4. **Standardized Certificate Management** - - - Integrates with cert-manager for automation - - Leverages established CA infrastructure - - Automatic certificate renewal (Let's Encrypt) - - 5. **Simplified Troubleshooting** - - - Standard diagnostic tools functional (curl, openssl s_client) - - Certificate chain validation is transparent - - DNS-based validation errors provide clear feedback - - **Limitations:** - - 1. **External CA Dependency** - - - Trust anchored in public or enterprise Certificate Authority - - Certificate revocation depends on CA infrastructure availability - - Not suitable for architectures requiring complete trust independence - - 2. **DNS Infrastructure Requirement** - - - Requires publicly resolvable DNS records - - DNS must resolve from all federation partner networks - - Vulnerable to DNS-based attacks (hijacking, poisoning) - - 3. **cert-manager Dependency** - - - Requires cert-manager installation and maintenance - - ClusterIssuer must be properly configured - - Additional component in the operational stack - - 4. **Limited Air-Gap Support** - - - Let's Encrypt requires internet connectivity for ACME challenges - - Private CA deployments require certificate distribution - - Increased complexity in isolated network environments - - **Recommended Use Cases:** - - - Cloud-native deployments (AWS, Google Cloud, Azure) - - Organizations with existing cert-manager infrastructure - - Environments with traffic inspection/WAF requirements - - Scenarios prioritizing rapid onboarding - - Public-facing federation endpoints - - Organizations restricting SSL passthrough - -=== "https_spiffe profile" - - Federation over SPIFFE mutual TLS using X.509-SVIDs issued by SPIRE. The SPIRE server presents a certificate containing a SPIFFE ID in the URI Subject Alternative Name (SAN), validated against a pre-shared trust bundle. This profile implements pure SPIFFE trust without external CA dependencies. - - **Architecture:** - - ```mermaid - flowchart TB - Remote[Remote SPIRE Bundle Consumer] - NGINX[NGINX Ingress Controller
• SSL Passthrough enabled
• Forwards encrypted traffic opaquely
• No inspection or termination] - SPIRE[SPIRE Server SPIFFE mTLS Bundle Provider] - - Remote -->|"SPIFFE mTLS (URI SAN + bootstrap)"| NGINX - NGINX -->|"Direct TLS passthrough"| SPIRE - ``` - - **Advantages:** - - 1. **Pure SPIFFE Trust Model** - - - No reliance on external Certificate Authorities - - Self-contained cryptographic identity verification - - Aligns with zero-trust architecture principles - - 2. **Air-Gap Environment Compatibility** - - - No internet connectivity required for operation - - Functions in completely isolated network environments - - DNS resolution optional (IP addresses supported) - - 3. **Eliminated cert-manager Dependency** - - - SPIRE manages certificate lifecycle internally - - Reduces operational components - - Simplified cluster requirements - - 4. **End-to-End Encryption Guarantee** - - - Traffic encrypted from source SPIRE to destination SPIRE - - No intermediate inspection or decryption points - - Maximum confidentiality assurance - - 5. **Strong Cryptographic Identity Binding** - - - Certificate cryptographically bound to specific SPIFFE ID - - Enhanced authenticity verification - - Federation partner explicitly identified and validated - - **Limitations:** - - 1. **SSL Passthrough Infrastructure Requirement** - - - Many enterprise environments restrict or prohibit SSL passthrough - - NGINX ingress controller requires special configuration - - Not universally supported across ingress implementations - - 2. **Bootstrap Bundle Coordination Required** - - - Manual exchange of initial trust bundles necessary - - Out-of-band communication channel required - - Coordination overhead between federation partners - - 3. **Traffic Inspection Incompatibility** - - - Web Application Firewalls cannot inspect traffic - - Load balancers lose request/response visibility - - DDoS protection limited to Layer 4 (transport layer) - - 4. **Increased Troubleshooting Complexity** - - - Standard diagnostic tools require SPIFFE certificates - - Error messages (e.g., "certificate contains no URI SAN") require SPIFFE expertise - - Steeper learning curve for operations teams - - 5. **Bootstrap Bundle Staleness Risk** - - - Federation breaks if CA rotates before bundle update - - Requires monitoring of bundle freshness - - Manual intervention required for trust bundle synchronization - - **Recommended Use Cases:** - - - Air-gapped or network-isolated environments - - Zero-trust architectures eliminating CA dependencies - - Organizations with SSL passthrough capabilities - - Scenarios requiring maximum security and confidentiality - - Pure SPIFFE/SPIRE architectural deployments - - Environments where traffic inspection is explicitly prohibited - -## Decision Matrix - -### Select the `https_web` profile if: - -- cert-manager is installed or can be deployed in your cluster -- NGINX ingress controller does not support or allow SSL passthrough -- Organizational security policies require traffic inspection or Web Application Firewall (WAF) -- Standard CA-based certificates are acceptable for your trust model -- Public DNS resolution is available for federation endpoints -- Operational priority is rapid deployment and simplified troubleshooting -- Deployment targets public cloud infrastructure (AWS, Google Cloud, Azure) - -### Select the `https_spiffe` profile if: - -- NGINX ingress controller supports and allows SSL passthrough configuration -- Deployment environment is air-gapped or network-isolated -- Zero-trust architecture requires elimination of external CA dependencies -- End-to-end encryption without intermediate inspection is mandatory -- Existing architecture is SPIFFE-centric -- Coordination for bootstrap bundle exchange is operationally feasible -- Organizational security model prioritizes pure cryptographic identity over CA trust - -### Default Recommendation - -For organizations without specific constraints, `https_web` is recommended as the default profile due to: - -- Broader infrastructure compatibility (no SSL passthrough requirement) -- Simplified initial setup (no bootstrap bundle coordination) -- Reduced operational complexity -- Compatibility with standard enterprise security controls - -## Bootstrap Bundle Exchange Process (https_spiffe) - -The bootstrap bundle exchange is a critical prerequisite for `https_spiffe` federation. This section provides detailed procedures for bundle generation, validation, and configuration. - -### Understanding Bootstrap Bundles - -A bootstrap bundle contains the public key material for a trust domain's Certificate Authority. The remote SPIRE server uses this bundle to validate the SPIFFE X.509-SVID presented by the federation endpoint during the initial TLS handshake. - -Key characteristics of bootstrap bundles: - -- Contains CA certificate and public keys (no private keys) -- Safe to transmit over untrusted channels -- Must be kept current as CAs rotate - -### Setting Up Bootstrap Bundles - -1. Extract Bundle from SPIRE Server - - There are three methods to extract the bootstrap bundle from the SPIRE server: - - === "Using kubectl exec (Recommended)" - - Connect to the SPIRE server pod and extract the bundle: - - ```bash - # Connect to SPIRE server pod and extract bundle - kubectl exec -n dir-spire deployment/spire-server -c spire-server -- \ - /opt/spire/bin/spire-server bundle show -format spiffe > my-trust-domain-bundle.json - - # Verify bundle contents - cat my-trust-domain-bundle.json - ``` - - ??? "Expected output format" - - ```json - { - "keys": [ - { - "use": "x509-svid", - "kty": "RSA", - "n": "xGOr-H7A-qw...", - "e": "AQAB", - "x5c": [ - "MIIC..." - ] - }, - { - "use": "jwt-svid", - "kty": "RSA", - "kid": "bJV3z...", - "n": "yH3s-K9B...", - "e": "AQAB" - } - ], - "spiffe_refresh_hint": 450000, - "spiffe_sequence_number": 1 - } - ``` - - === "Using spire-server CLI Locally" - - If you have the SPIRE server binary installed locally and configured with access to the SPIRE server: - - ```bash - # Show bundle in SPIFFE format - spire-server bundle show \ - -format spiffe \ - -socketPath /run/spire/sockets/server.sock \ - > my-trust-domain-bundle.json - ``` - - === "Via Federation Endpoint" - - Once your federation endpoint is operational, bundles can be fetched via HTTPS: - - ```bash - # Fetch bundle from federation endpoint - curl -k https://your-spire.example.com > my-trust-domain-bundle.json - ``` - -2. Validate the bundle structure to ensure it is well-formed: - - ```bash - # Verify JSON is well-formed - jq '.' my-trust-domain-bundle.json - - # Check for required x509-svid key - jq '.keys[] | select(.use == "x509-svid")' my-trust-domain-bundle.json - - # Verify key count (should have at least one x509-svid key) - jq '.keys | length' my-trust-domain-bundle.json - - # Optionally decode and inspect the X.509 certificate - jq -r '.keys[] | select(.use == "x509-svid") | .x5c[0]' my-trust-domain-bundle.json | \ - base64 -d | \ - openssl x509 -inform DER -text -noout - ``` - -3. Add the bundle to your ClusterFederatedTrustDomain resource configuration: - - ```yaml - # onboarding/federation/partner.com.yaml - className: dir-spire - trustDomain: partner.com - bundleEndpointURL: https://spire.partner.com - bundleEndpointProfile: - type: https_spiffe - endpointSPIFFEID: spiffe://partner.com/spire/server - trustDomainBundle: |- - { - "keys": [ - { - "use": "x509-svid", - "kty": "RSA", - "n": "xGOr-H7A-qw...", - "e": "AQAB", - "x5c": ["MIIC..."] - }, - { - "use": "jwt-svid", - "kty": "RSA", - "kid": "bJV3z...", - "n": "yH3s-K9B...", - "e": "AQAB" - } - ], - "spiffe_refresh_hint": 450000, - "spiffe_sequence_number": 1 - } - ``` - -!!! note - Organizations must establish their own secure procedures for exchanging bootstrap bundles with federation partners. The bundle exchange mechanism (email, file transfer, version control, etc.) should align with organizational security policies. diff --git a/docs/dir/federation-troubleshooting.md b/docs/dir/federation-troubleshooting.md deleted file mode 100644 index 18b5e24..0000000 --- a/docs/dir/federation-troubleshooting.md +++ /dev/null @@ -1,236 +0,0 @@ -# Federation Best Practices and Troubleshooting - -This page covers operational best practices and troubleshooting for Directory federation. It applies to both [https_web and https_spiffe](federation-profiles.md#profile-comparison) profiles. For profile selection and configuration details, see [Federation Bundle Profiles](federation-profiles.md). - -## Operational Best Practices - -### Universal Best Practices - -- Consistent className configuration - - - Use standardized className value across all resources (recommended: "dir-spire") - - Ensures SPIRE Controller Manager correctly processes ClusterSPIFFEID resources - - Prevents controller filtering issues - -- Enable ClusterFederatedTrustDomain Controller - - - Required for automatic trust bundle fetching - - Without this controller, federation configuration is ignored - - Verify in controller manager logs - -- Implement Bundle Refresh Monitoring - - - Monitor SPIRE server logs for "Bundle refreshed" events - - Alert on bundle staleness exceeding 24 hours - - Implement automated monitoring: - - ```bash - kubectl logs -n dir-spire -l app.kubernetes.io/name=server -c spire-server | \ - grep "Bundle refreshed" | \ - grep "trust_domain=partner.com" - ``` - -- Configure Rate Limiting - - - Protect federation endpoints from abuse - - Recommended configuration: 100 requests/second with 5x burst multiplier - - Adjust based on number of federation partners and refresh frequency - -- Set Appropriate CA TTL - - - Recommended: 720 hours (30 days) - - Balances security (shorter TTL) with operational stability (fewer rotations) - - Consider operational capacity for handling rotation issues - -### https_web Specific Practices - -- Use Production Certificate Authority - - - Deploy with production Let's Encrypt (`letsencrypt` ClusterIssuer) - - Avoid staging certificates in production federation scenarios - - Staging certificates are not trusted by remote SPIRE servers by default - -- Monitor Certificate Expiration - - - Although cert-manager auto-renews, implement monitoring - - Alert on certificates expiring within 7 days - - Verify renewal process during certificate lifecycle: - - ```bash - kubectl get certificate -n dir-spire spire-server-federation-cert -o yaml | \ - grep -A5 "status:" - ``` - -- Validate External Network Accessibility - - - Periodically test federation endpoint from external network - - Verify DNS resolution from federation partner networks - - Validate certificate chain visibility: - - ```bash - curl -v https://spire.example.com 2>&1 | grep -A10 "SSL certificate" - ``` - -### https_spiffe Specific Practices - -- Maintain Bootstrap Bundle Freshness - - - Update bootstrap bundles before CA rotation (every caTTL period) - - Implement bundle distribution automation where feasible - - Document manual update procedures for operational staff - -- Document Bundle Exchange Procedures - - - Establish clear onboarding process for new federation partners - - Define secure channels for bundle transmission - - Maintain contact registry for federation partner technical contacts - -- Validate SSL Passthrough Persistence - - - Verify ssl-passthrough configuration after cluster maintenance - - Test after NGINX ingress controller upgrades - - Automate validation: - - ```bash - kubectl get deployment -n ingress-nginx ingress-nginx-controller -o yaml | \ - grep "enable-ssl-passthrough" || echo "WARNING: SSL passthrough not enabled" - ``` - -## Troubleshooting - -### Connection Issues - -```bash -# Check SPIRE agent status -spire-agent api fetch x509-svid - -# Verify network connectivity -curl -v https://prod.api.ads.outshift.io - -# Verify env vars are set -echo $DIRECTORY_CLIENT_SERVER_ADDRESS $DIRECTORY_CLIENT_SPIFFE_SOCKET_PATH -``` - -### Federation Issues - -```bash -# Verify trust bundle exchange -spire-server federation show --trustDomain prod.ads.outshift.io - -# Test bundle endpoint -curl https://prod.spire.ads.outshift.io/ -``` - -### Common Errors - -| Error | Solution | -|-------|----------| -| `connection refused` | Check SPIRE agent running and socket path | -| `x509: certificate signed by unknown authority` | Verify trust bundle configuration | -| `context deadline exceeded` | Check network and firewall | -| `permission denied` | Ensure SPIFFE ID registration and policies | - -### https_web Profile Issues - -#### Issue: "certificate is valid for ingress.local, not spire.example.com" - -**Root Cause:** NGINX serving default certificate instead of cert-manager issued certificate. - -**Resolution:** Ensure `tlsSecret` is set under `spire-server.ingress` in your Helm values: - -```yaml -# Under spire-server.ingress in values -ingress: - tlsSecret: "spire-server-federation-cert" -``` - -**Verification:** -```bash -kubectl get certificate -n dir-spire spire-server-federation-cert -# Status should be READY=True -``` - -#### Issue: "certificate signed by unknown authority" - -**Root Cause:** cert-manager certificate not issued or ClusterIssuer misconfigured. - -**Resolution:** -```bash -# Check Certificate status -kubectl describe certificate -n dir-spire spire-server-federation-cert - -# Check ClusterIssuer status -kubectl get clusterissuer letsencrypt -o yaml - -# Review cert-manager logs -kubectl logs -n cert-manager deployment/cert-manager -``` - -#### Issue: "no endpoints available for service" - -**Root Cause:** Ingress not routing traffic to SPIRE server service. - -**Resolution:** -```bash -# Verify Ingress status -kubectl get ingress -n dir-spire - -# Check service endpoints -kubectl get endpoints -n dir-spire spire-server - -# Verify SPIRE server pods are running -kubectl get pods -n dir-spire -l app.kubernetes.io/name=server -``` - -### https_spiffe Profile Issues - -#### Issue: "certificate contains no URI SAN" - -**Root Cause:** SPIRE server has not issued itself a federation X.509-SVID. - -**Resolution:** -```bash -# Restart SPIRE server to trigger SVID issuance -kubectl rollout restart -n dir-spire deployment/spire-server - -# Wait for rollout to complete -kubectl rollout status -n dir-spire deployment/spire-server - -# Verify federation SVID issuance in logs -kubectl logs -n dir-spire -l app.kubernetes.io/name=server -c spire-server | \ - grep "federation" -``` - -#### Issue: "certificate signed by unknown authority" - -**Root Cause:** Bootstrap bundle is missing, stale, or incorrectly formatted. - -**Resolution:** Obtain a fresh bundle from your federation partner, validate it, and update `trustDomainBundle` in your federation resource. See [Bootstrap Bundle Exchange Process](federation-profiles.md#bootstrap-bundle-exchange-process-https_spiffe) for extraction and validation steps: - -```bash -# Obtain fresh bundle from federation partner -curl -k https://spire.partner.com > fresh-partner-bundle.json - -# Validate bundle structure -jq '.' fresh-partner-bundle.json - -# Update trustDomainBundle in federation configuration -# Redeploy application -``` - -#### Issue: "ssl-passthrough annotation not taking effect" - -**Root Cause:** NGINX ingress controller not configured with `--enable-ssl-passthrough` flag. - -**Resolution:** -```bash -# Verify controller has ssl-passthrough enabled -kubectl get deployment -n ingress-nginx ingress-nginx-controller -o yaml | \ - grep "enable-ssl-passthrough" - -# If not present, patch deployment -kubectl patch deployment -n ingress-nginx ingress-nginx-controller --type='json' \ - -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value":"--enable-ssl-passthrough"}]' - -# Verify configuration persisted after controller restart -``` diff --git a/docs/dir/getting-started.md b/docs/dir/getting-started.md deleted file mode 100644 index 3109ecb..0000000 --- a/docs/dir/getting-started.md +++ /dev/null @@ -1,343 +0,0 @@ -# Getting Started - -Deploy Directory with SPIRE in a Kind cluster for development and testing. Uses `example.org` as the trust domain (local only—cannot federate with the public network). - -## Prerequisites - -The following prerequisites are required: - -- [Docker](https://www.docker.com/) -- [Kind](https://kind.sigs.k8s.io/) -- [kubectl](https://kubernetes.io/docs/tasks/tools/) -- [Helm](https://helm.sh/) 3.x - -!!! note - Make sure that Docker has Buildx enabled. - -## Installation - -The Agent Directory Service can be deployed using Helm or GitOps / Argo CD. Helm is the recommended method for development and testing. - -=== "Helm" - - Deploy Directory using Helm only—no Argo CD. Uses the same charts and configuration patterns as [dir-staging](https://github.com/agntcy/dir-staging). - - 1. Create Kind cluster: - - ```bash - kind create cluster --name dir-dev - ``` - - 2. Add Helm repositories: - - ```bash - helm repo add spiffe https://spiffe.github.io/helm-charts-hardened - helm repo update - ``` - - 3. Deploy SPIRE CRDs: - - ```bash - helm install spire-crds spiffe/spire-crds \ - --version 0.5.0 \ - --namespace dir-dev-spire-crds \ - --create-namespace - ``` - - 4. Deploy SPIRE server and agent: - - ```bash - helm install spire spiffe/spire \ - --version 0.27.0 \ - --namespace dir-dev-spire \ - --create-namespace \ - --set global.spire.trustDomain=example.org \ - --set global.installAndUpgradeHooks.enabled=false \ - --set global.deleteHooks.enabled=false \ - --set spire-server.federation.enabled=true \ - --set spire-server.controllerManager.className=dir-spire - ``` - - Wait for SPIRE to be ready: - - ```bash - kubectl wait --for=condition=ready pod -n dir-dev-spire -l app.kubernetes.io/name=server --timeout=240s - kubectl wait --for=condition=ready pod -n dir-dev-spire -l app.kubernetes.io/name=agent --timeout=240s - ``` - - 5. Deploy Directory (API server, Zot, PostgreSQL) - - The Directory chart includes the API server, Zot registry, and PostgreSQL. Pipe values from stdin using `-f -`: - - ```bash - helm install dir oci://ghcr.io/agntcy/dir/helm-charts/dir \ - --version v1.0.0 \ - --namespace dir-dev-dir \ - --create-namespace \ - -f - <<'EOF' - # Minimal values for Kind - based on dir-staging applications/dir/dev/values.yaml - # Release name: dir. Service names: dir-apiserver, dir-zot (chart.oci.registryAddress uses Release.Name-zot) - apiserver: - image: - repository: ghcr.io/agntcy/dir-apiserver - tag: v1.0.0 - pullPolicy: IfNotPresent - service: - type: NodePort - metrics: - enabled: false - routingService: - type: NodePort - nodePort: 30555 - spire: - enabled: true - className: dir-spire - trustDomain: example.org - useCSIDriver: true - config: - listen_address: "0.0.0.0:8888" - oasf_api_validation: - disable: true - authn: - enabled: true - mode: "x509" - socket_path: "unix:///run/spire/agent-sockets/api.sock" - audiences: - - "spiffe://example.org/spire/server" - authz: - enabled: true - enforcer_policy_file_path: "/etc/agntcy/dir/authz_policies.csv" - ratelimit: - enabled: false - store: - provider: "oci" - oci: - registry_address: "dir-zot.dir-dev-dir.svc.cluster.local:5000" - auth_config: - insecure: "true" - username: "admin" - password: "admin" - routing: - listen_address: "/ip4/0.0.0.0/tcp/5555" - datastore_dir: /etc/routing/datastore - directory_api_address: "dir-apiserver.dir-dev-dir.svc.cluster.local:8888" - gossipsub: - enabled: false - sync: - auth_config: - username: "user" - password: "user" - publication: - scheduler_interval: "1h" - worker_count: 0 - worker_timeout: "30m" - database: - type: "postgres" - postgres: - host: "" - port: 5432 - database: "dir" - ssl_mode: "disable" - authz_policies_csv: | - p,example.org,* - p,*,/agntcy.dir.store.v1.StoreService/Pull - p,*,/agntcy.dir.store.v1.StoreService/PullReferrer - p,*,/agntcy.dir.store.v1.StoreService/Lookup - p,*,/agntcy.dir.store.v1.SyncService/RequestRegistryCredentials - secrets: - ociAuth: - username: "admin" - password: "admin" - reconciler: - config: - regsync: - enabled: false - indexer: - enabled: false - postgresql: - enabled: true - auth: - username: "dir" - password: "dir" - database: "dir" - strategy: - type: Recreate - extraVolumeMounts: - - name: dir-routing-storage - mountPath: /etc/routing - extraVolumes: - - name: dir-routing-storage - emptyDir: {} - zot: - resources: {} - mountSecret: true - authHeader: "admin:admin" - secretFiles: - htpasswd: |- - admin:$2y$05$vmiurPmJvHylk78HHFWuruFFVePlit9rZWGA/FbZfTEmNRneGJtha - user:$2y$05$L86zqQDfH5y445dcMlwu6uHv.oXFgT6AiJCwpv3ehr7idc0rI3S2G - mountConfig: true - configFiles: - config.json: |- - { - "distSpecVersion": "1.1.1", - "storage": {"rootDirectory": "/var/lib/registry"}, - "http": { - "address": "0.0.0.0", - "port": "5000", - "auth": {"htpasswd": {"path": "/secret/htpasswd"}}, - "accessControl": { - "adminPolicy": {"users": ["admin"], "actions": ["read", "create", "update", "delete"]}, - "repositories": {"**": {"anonymousPolicy": [], "defaultPolicy": ["read"]}} - } - }, - "log": {"level": "info"}, - "extensions": {"search": {"enable": true}, "trust": {"enable": true, "cosign": true, "notation": false}} - } - EOF - ``` - - 6. Wait for the API server to be ready: - - ```bash - kubectl wait --for=condition=ready pod -n dir-dev-dir -l app.kubernetes.io/name=apiserver --timeout=240s - ``` - - 7. Port-forward and verify: - - ```bash - kubectl port-forward svc/dir-apiserver -n dir-dev-dir 8888:8888 - ``` - - 8. In another terminal, verify with token-based auth: - - ```bash - # Install dirctl (if needed) - brew tap agntcy/dir https://github.com/agntcy/dir - brew install dirctl - - # Create SPIFFE SVID for local client - SPIRE_POD=$(kubectl get pods -n dir-dev-spire -l app.kubernetes.io/name=server -o jsonpath='{.items[0].metadata.name}') - kubectl exec -n dir-dev-spire "$SPIRE_POD" -c spire-server -- \ - /opt/spire/bin/spire-server x509 mint \ - -dns dev.api.example.org \ - -spiffeID spiffe://example.org/local-client \ - -output json > spiffe-dev.json - - # Configure and verify - export DIRECTORY_CLIENT_AUTH_MODE="token" - export DIRECTORY_CLIENT_SPIFFE_TOKEN="spiffe-dev.json" - export DIRECTORY_CLIENT_SERVER_ADDRESS="127.0.0.1:8888" - export DIRECTORY_CLIENT_TLS_SKIP_VERIFY="true" - - dirctl info bafytest123 - # Expected: Error: record not found (proves API is reachable) - ``` - - 9. Push and pull a record: - - ```bash - # Push via pipe - cat <<'EOF' | dirctl push --stdin --output raw - {"name":"burger_seller_agent","schema_version":"1.0.0","version":"1.0.0","description":"Helps with creating burger orders","authors":["Example Organization"],"created_at":"2025-01-01T00:00:00Z","skills":[{"name":"natural_language_processing/natural_language_understanding/contextual_comprehension","id":10101}],"locators":[{"type":"container_image","urls":["https://ghcr.io/agntcy/burger-seller-agent"]}],"modules":[{"name":"integration/mcp","data":{"name":"github-mcp-server","connections":[{"type":"stdio","command":"docker","args":["run","-i","--rm","-e","GITHUB_PERSONAL_ACCESS_TOKEN","ghcr.io/github/github-mcp-server"],"env_vars":[{"name":"GITHUB_PERSONAL_ACCESS_TOKEN","default_value":"","description":"Secret value for GITHUB_PERSONAL_ACCESS_TOKEN"}]}]}},{"name":"integration/a2a","data":{"card_data":{"protocolVersions":["0.2.6"],"name":"burger_seller_agent","description":"Helps with creating burger orders","supportedInterfaces":[{"url":"https://burger-agent-109790610330.us-central1.run.app","protocolBinding":"HTTP+JSON"}],"provider":{"url":"https://example.com","organization":"Example Organization"},"version":"1.0.0","capabilities":{"streaming":true},"defaultInputModes":["text","text/plain"],"defaultOutputModes":["text","text/plain"],"skills":[{"id":"create_burger_order","name":"Burger Order Creation Tool","description":"Helps with creating burger orders","tags":["burger order creation"],"examples":["I want to order 2 classic cheeseburgers"]}]},"card_schema_version":"v1.0.0"}}]} - EOF - - # Pull by name (JSON output) - dirctl pull burger_seller_agent -o json - ``` - - 10. Clean up the environment: - - ```bash - helm uninstall dir -n dir-dev-dir - helm uninstall spire -n dir-dev-spire - helm uninstall spire-crds -n dir-dev-spire-crds - kind delete cluster --name dir-dev - ``` - - **Chart References** - - | Component | Chart | Version | - |-----------|-------|---------| - | **SPIRE CRDs** | `spiffe/spire-crds` | 0.5.0 | - | **SPIRE** | `spiffe/spire` | 0.27.0 | - | **Directory** | `oci://ghcr.io/agntcy/dir/helm-charts/dir` | v1.0.0 | - - For full configuration examples see the [dir-staging applications](https://github.com/agntcy/dir-staging/tree/main/applications). - -=== "GitOps / Argo CD" - - For teams already using Argo CD, Directory can be deployed from a GitOps repository. [dir-staging](https://github.com/agntcy/dir-staging) is an example you can fork or clone to maintain your own GitOps repo. - - 1. Create a Kind cluster and install Argo CD. - 2. Apply the dir-staging project and application manifests (or your forked repo). - 3. Argo CD syncs SPIRE CRDs, SPIRE, and Directory from the repo. - - ```bash - kind create cluster --name dir-dev - kubectl create namespace argocd - kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml - kubectl wait --namespace argocd --for=condition=available deployment --all --timeout=120s - - kubectl apply -f https://raw.githubusercontent.com/agntcy/dir-staging/main/projects/dir/dev/dir-dev.yaml - kubectl apply -f https://raw.githubusercontent.com/agntcy/dir-staging/main/projectapps/dir/dev/dir-dev-projectapp.yaml - - kubectl wait --for=condition=ready pod -n dir-dev-spire -l app.kubernetes.io/name=server --timeout=240s - kubectl wait --for=condition=ready pod -n dir-dev-spire -l app.kubernetes.io/name=agent --timeout=240s - kubectl wait --for=condition=ready pod -n dir-dev-dir -l app.kubernetes.io/name=apiserver --timeout=240s - ``` - - Port-forward: `kubectl port-forward svc/dir-dir-dev-argoapp-apiserver -n dir-dev-dir 8888:8888` - - See [dir-staging](https://github.com/agntcy/dir-staging) for full configuration and customization options. - - !!! important "Trust domain" - This Quick Start uses `example.org` for local testing only. To federate with the public Directory network, you need a unique trust domain. See [Production Deployment](prod-deployment.md) and [Running a Federated Directory Instance](partner-prod-federation.md). - -## Deployment - -Directory API services can be deployed either using the Taskfile, Docker Compose, or directly via the released Helm chart. - -=== "Taskfile" - - Start the necessary components (such as storage and API services): - - ```bash - DIRECTORY_SERVER_OASF_API_VALIDATION_SCHEMA_URL=https://schema.oasf.outshift.com task server:start - ``` - - !!! note - - MacOS users may encounter a "port 5000 already in use" error. This is likely caused by the AirPlay Receiver feature. You can disable it in your System Settings. - -=== "Docker Compose" - - This method deploys Directory services using Docker Compose. - - While the Directory server has a default OASF schema URL, Docker Compose deployments may require explicitly setting the `DIRECTORY_SERVER_OASF_API_VALIDATION_SCHEMA_URL` environment variable: - - ```bash - cd install/docker - export DIRECTORY_SERVER_OASF_API_VALIDATION_SCHEMA_URL=https://schema.oasf.outshift.com - docker compose up -d - ``` - - Alternatively, you can disable API validation (not recommended for production): - - ```bash - cd install/docker - export DIRECTORY_SERVER_OASF_API_VALIDATION_DISABLE=true - docker compose up -d - ``` - -For more configuration options, see [Validation](validation.md). - -## Directory MCP Server - -The Directory services are also accessible through the Directory MCP Server. It provides a standardized interface for AI assistants and tools to interact with the Directory system and work with OASF agent records. See the [MCP Server documentation](directory-mcp.md) for more information. - -## Next Steps - -- Connect to the public Directory: federate with the public Directory network at `prod.api.ads.outshift.io` to discover and publish agents. See [Running a Federated Directory Instance](partner-prod-federation.md). -- Use the [Directory CLI](directory-cli.md) to create and query records. -- Explore [Features and Usage Scenarios](scenarios.md): build, store, sign, discover, search. diff --git a/docs/dir/overview.md b/docs/dir/overview.md deleted file mode 100644 index 8b84fe4..0000000 --- a/docs/dir/overview.md +++ /dev/null @@ -1,144 +0,0 @@ -# Overview - -The Agent Directory Service (ADS) is a distributed directory service designed to -store metadata for AI agent applications. This metadata, stored as directory -records, enables the discovery of agent applications with specific skills for -solving various problems. -The implementation features distributed directories that interconnect through a -content-routing protocol. This protocol maps agent skills to directory record -identifiers and maintains a list of directory servers currently hosting those -records. -Directory records are identified by globally unique names that are routable -within a DHT (Distributed Hash Table) to locate peer directory servers. -Similarly, the skill taxonomy is routable in the DHT to map skillsets to records -that announce those skills. - -The Agent Directory leverages the [OASF](../oasf/open-agentic-schema-framework.md) to -describe AI agents and provides a set of APIs and tools to build, store, publish -and discover AI agents across the network by their attributes and constraints. -Directory also leverages the [CSIT](../csit/csit.md) for continuous system -integration and testing across different versions, environments, and features. - -Each directory record must include skills from a defined taxonomy, as specified -in the [Taxonomy of AI Agent Skills](https://schema.oasf.outshift.com/skill_categories) from the [OASF](../oasf/open-agentic-schema-framework.md). -While all record data is modeled using the [OASF](../oasf/open-agentic-schema-framework.md), only skills are -leveraged for content routing in the distributed network of directory servers. -The ADS specification is under active development and is published as an -Internet Draft at [ADS Spec](https://spec.dir.agntcy.org). The source code is -available in the [ADS Spec sources](https://github.com/agntcy/dir-spec). -The current reference implementation, written in Go, provides server and client -nodes with gRPC and protocol buffer interfaces. The directory record storage is -built on [ORAS](https://oras.land) (OCI Registry As Storage), while data -distribution uses the [zot](https://zotregistry.dev) OCI server implementation. - -## Features - -ADS enables several key capabilities for the agentic AI ecosystem: - -- **Capability-Based Discovery**: Agents publish structured metadata describing their -functional characteristics as described by the [OASF](../oasf/open-agentic-schema-framework.md). -The system organizes this information using hierarchical taxonomies, -enabling efficient matching of capabilities to requirements. -- **Verifiable Claims**: While agent capabilities are often subjectively evaluated, -ADS provides cryptographic mechanisms for data integrity and provenance tracking. -This allows users to make informed decisions about agent selection. -- **Semantic Linkage**: Components can be securely linked to create various relationships -like version histories for evolutionary development, collaborative partnerships where -complementary skills solve complex problems, and dependency chains for composite agent workflows. -- **Distributed Architecture**: Built on proven distributed systems principles, -ADS uses content-addressing for global uniqueness and implements distributed hash tables (DHT) -for scalable content discovery across decentralized networks. -- **Runtime Discovery**: Automatically discover and query agent workloads running in container -runtimes (Docker, Kubernetes). The discovery system watches for labeled containers/pods, -resolves their metadata (A2A cards, OASF records), and exposes them via gRPC API for -integration with other Directory components. - -## Naming - -In distributed systems, having a reliable and collision-resistant naming scheme -is crucial. The agent directory uses cryptographic hashes to generate globally -unique identifiers for data records. -ADS leverages [Content Identifiers](https://github.com/multiformats/cid) for -naming directory records. CIDs provide a self-describing, content-addressed -naming scheme that ensures data integrity and immutability. - -In addition to CID-based addressing, ADS supports verifiable domain-based names that enable human-readable references while maintaining cryptographic verification. See the [Directory CLI documentation](directory-cli.md#name-verification) for details. - -## Content Routing - -ADS implements capability-based record discovery through a hierarchical skill -taxonomy. This architecture enables: - -1. Capability Announcement: - 1. Multi-agent systems can publish their capabilities by encoding them as skill taxonomies. - 2. Each record contains metadata describing the agent's functional capabilities. - 3. Skills are structured in a hierarchical format for efficient matching. -2. Discovery Process: The system performs a two-phase discovery operation: - 1. Matches queried capabilities against the skill taxonomy to determine records by their identifier. - 2. Identifies the server nodes storing relevant records. -3. Distributed Resolution: Local nodes execute targeted retrievals based on: - 1. Skill matching results: Evaluates capability requirements. - 2. Server location information: Determines optimal data sources. - -ADS uses [libp2p Kad-DH](https://github.com/libp2p/specs/tree/master/kad-dht) for server and content discovery. - -## Distributed Object Storage - -ADS differs from block storage systems like -[IPFS](https://ipfs.tech/) in its approach to distributed object storage. -The differences are described in the following sections. - -### Simplified Content Retrieval - -1. ADS directly stores complete records rather than splitting them into blocks. -2. No special optimizations needed for retrieving content from multiple sources. -3. Records are retrieved as complete units using standard OCI protocols. - -### OCI Integration - -ADS leverages the OCI distribution specification for content storage and retrieval: - -1. Records are stored and transferred using OCI artifacts. -2. Any OCI distribution-compliant server can participate in the network. -3. Servers retrieve records directly from each other using standard OCI protocols. - -While ADS uses [zot](https://zotregistry.dev) as its reference OCI server implementation, the system works -with any server that implements the OCI distribution specification. - -## Flow Diagrams - -```mermaid -sequenceDiagram - participant User - participant DHT - participant ServerA - participant ServerB - participant ServerC - - Note over ServerA,ServerC: Publication Phase - ServerA->>ServerA: Generate record CID - ServerA->>ServerA: Extract skills from record - ServerA->>ServerA: Store record locally - ServerA->>DHT: Announce CID + skills - ServerB->>ServerB: Generate record CID - ServerB->>ServerB: Extract skills from record - ServerB->>ServerB: Store record locally - ServerB->>DHT: Announce CID + skills - DHT->>DHT: Update routing tables
(skills→CIDs→servers) - - Note over User,ServerC: Discovery Phase - User->>DHT: Query by skills - DHT->>DHT: Search routing tables - DHT->>User: Return matching CIDs
+ server addresses - User->>User: Select records - User->>ServerA: Download record 1 - User->>ServerB: Download record 2 -``` - -## Next Steps - -Ready to get started? Choose your path: - -- Run a local instance: deploy Directory with SPIRE in a Kind cluster for development and testing. See [Getting Started](getting-started.md). -- Connect to the public Directory: use the existing network at `prod.api.ads.outshift.io` to discover and publish agents. See [Running a Federated Directory Instance](partner-prod-federation.md). -- Deploy for production: run your own Directory instance on AWS EKS and optionally federate with the network. See [Production Deployment](prod-deployment.md). diff --git a/docs/dir/partner-prod-federation.md b/docs/dir/partner-prod-federation.md deleted file mode 100644 index f24d858..0000000 --- a/docs/dir/partner-prod-federation.md +++ /dev/null @@ -1,302 +0,0 @@ -# Running a Federated Directory Instance - -This guide explains how to federate your Directory instance (`partner.io`) with the public production Directory at `prod.ads.outshift.io`. The prod instance uses the `https_web` bundle endpoint profile (Let's Encrypt, standard HTTPS). Your instance must use `https_web` as well for compatibility. - -Partnering with the Production Directory involves two trust domains. `partner.io` is your instance's trust domain and `prod.ads.outshift.io` is the production Directory's trust domain. - -Assumptions: - -- Your endpoints are publicly available: `spire.partner.io`, `oidc-discovery.spire.partner.io`, `zot.partner.io`, `api.partner.io`. -- Let's Encrypt production issuer with cert-manager is deployed in your cluster (`letsencrypt-prod` or equivalent). -- NGINX (or compatible) Ingress controller is available. - -## Prerequisites - -- Kubernetes cluster with an Ingress controller. -- cert-manager with a Let's Encrypt production issuer (`letsencrypt-prod` or equivalent). -- Public DNS records pointing to your Ingress (or LoadBalancer). - -## Setting up the Federation - -1. Deploy SPIRE with https_web federation - - SPIRE must use the `https_web` profile so it can fetch prod's bundle over standard HTTPS (Let's Encrypt). The prod federation endpoint is `https://prod.spire.ads.outshift.io`. - - ??? example "Deploy SPIRE with https_web federation" - - ```bash - helm repo add spiffe https://spiffe.github.io/helm-charts-hardened/ - helm repo update - - helm upgrade --install --create-namespace -n spire-crds spire-crds spire-crds \ - --repo https://spiffe.github.io/helm-charts-hardened/ - - helm upgrade --install -n spire spire spire \ - --repo https://spiffe.github.io/helm-charts-hardened/ \ - -f - <<'EOF' - global: - spire: - trustDomain: partner.io - clusterName: partner - namespaces: - create: false - ingressControllerType: other - - installAndUpgradeHooks: - enabled: false - deleteHooks: - enabled: false - - spire-server: - federation: - enabled: true - tls: - spire: - enabled: false - certManager: - enabled: true - issuer: - create: false - certificate: - issuerRef: - kind: ClusterIssuer - name: letsencrypt-prod - ingress: - enabled: true - className: nginx - controllerType: other - host: spire.partner.io - tlsSecret: spire-partner-federation-cert - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod - external-dns.alpha.kubernetes.io/hostname: spire.partner.io - nginx.ingress.kubernetes.io/ssl-passthrough: "false" - nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" - nginx.ingress.kubernetes.io/proxy-ssl-server-name: "on" - nginx.ingress.kubernetes.io/proxy-ssl-name: "spire.partner.io" - nginx.ingress.kubernetes.io/proxy-ssl-verify: "off" - controllerManager: - watchClassless: true - className: dir-spire - identities: - clusterFederatedTrustDomain: - enabled: true - clusterSPIFFEIDs: - default: - federatesWith: - - prod.ads.outshift.io - - spiffe-oidc-discovery-provider: - ingress: - enabled: true - className: nginx - host: oidc-discovery.spire.partner.io - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod - external-dns.alpha.kubernetes.io/hostname: oidc-discovery.spire.partner.io - config: - domains: - - oidc-discovery.spire.partner.io - EOF - ``` - - !!! note - Adjust `letsencrypt-prod` to match your ClusterIssuer name. Use `letsencrypt` if that is your production issuer. - -2. Deploy the Directory - - Deploy the Directory chart with SPIRE enabled and federation to prod. The dir chart creates `ClusterFederatedTrustDomain` resources from `apiserver.spire.federation`; the SPIRE server will fetch prod's bundle from the configured endpoint (prod uses `https_web`—standard HTTPS, no bootstrap bundle required). - - ??? example "Deploy Directory using the chart" - - ```bash - helm install dir oci://ghcr.io/agntcy/dir/helm-charts/dir \ - --version v1.0.0 \ - --namespace dir \ - --create-namespace \ - -f - <api.your-domain.com
zot.your-domain.com
spire.your-domain.com] - LB[AWS Network LB
Layer 3/4 TCP] - NGINX[NGINX Ingress Controller
SSL Passthrough API / TLS Termination Zot] - Dir[dir-apiserver SPIFFE] - Zot[Zot Registry] - Admin[dir-admin CronJobs] - - Clients --> DNS --> LB --> NGINX - NGINX --> Dir - NGINX --> Zot - Dir --> Admin -``` - -## Prerequisites - -### Infrastructure - -- **AWS EKS Cluster** – Kubernetes 1.31+ -- **NGINX Ingress Controller** with `--enable-ssl-passthrough=true` -- **SPIRE** – SPIFFE Runtime Environment -- **ExternalDNS** – Automatic DNS management (Route53) -- **cert-manager** – TLS certificate management -- **External Secrets Operator** – Vault integration -- **ArgoCD** – GitOps deployment - -### Network - -- **Route53 Hosted Zone** – For your domains (e.g., `*.your-domain.com`) -- **AWS Network Load Balancer** – Layer 3/4 TCP passthrough -- **Security Groups** – Allow cluster NAT Gateway IP for ingress - -### Storage - -- **EBS CSI Driver** – For persistent volumes -- **Storage Class** – `ebs-sc-encrypted` for production -- **Vault** – For credential storage - -## Local vs Production - -| Feature | Local (Kind) | Production (EKS) | -|---------|--------------|------------------| -| **Cluster** | Kind | AWS EKS | -| **SPIFFE CSI Driver** | ✅ Enabled | ✅ Enabled | -| **Storage** | emptyDir (ephemeral) | PVCs (persistent) | -| **Credentials** | Hardcoded in values | ExternalSecrets + Vault | -| **Resources** | 250m/512Mi | 500m–2000m / 1–4Gi | -| **Ingress** | NodePort, port-forward | Ingress + TLS | -| **Rate Limits** | 50 RPS | 500+ RPS | -| **Trust Domain** | example.org (local only) | your-domain.com | - -## Key Production Features - -### SPIFFE CSI Driver - -Enabled via `spire.useCSIDriver: true` (v1.0.0-rc.4+): - -- Synchronous workload identity injection before pod start -- Eliminates "certificate contains no URI SAN" errors -- Required for CronJobs and short-lived workloads - -### Persistent Storage - -- Enable PVCs for routing datastore and database -- Use `strategy.type: Recreate` to prevent database lock conflicts -- Production example: 20Gi routing, 5Gi database, 100Gi Zot - -### Secure Credential Management - -- Use External Secrets Operator with Vault -- See [External Secrets Operator documentation](https://external-secrets.io/latest/) - -### SSL Passthrough for DIR API - -The Directory API uses SPIFFE mTLS. Ingress must: - -1. **Not** terminate TLS -2. Pass encrypted traffic to the backend -3. Route based on SNI - -**Ingress configuration:** - -```yaml -annotations: - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - nginx.ingress.kubernetes.io/backend-protocol: "GRPCS" - nginx.ingress.kubernetes.io/force-ssl-redirect: "true" - -tls: - - hosts: - - api.your-domain.com - # NO secretName - required for SSL passthrough! -``` - -**NGINX Ingress Controller** must have `--enable-ssl-passthrough=true`. - -## DNS Hostnames - -Create DNS records for your domain. Example with `your-domain.com`: - -| Service | Hostname | -|---------|----------| -| **Directory API** | api.your-domain.com | -| **Zot Registry** | zot.your-domain.com | -| **SPIRE Federation** | spire.your-domain.com | -| **SPIRE OIDC** | oidc-discovery.spire.your-domain.com | - -## Verification - -### SSL Passthrough - -```bash -# Should show SPIFFE certificate, not "ingress.local" -echo | openssl s_client -connect api.your-domain.com:443 \ - -servername api.your-domain.com 2>/dev/null | \ - openssl x509 -noout -subject - -# Expected: C=US, O=SPIRE, CN=api.your-domain.com (or your trust domain) -``` - -### SPIFFE Authentication - -```bash -kubectl logs -n -l app.kubernetes.io/name=apiserver | \ - grep "Successfully obtained valid X509-SVID" -``` - -### CronJobs - -```bash -kubectl get pods -n --sort-by=.metadata.creationTimestamp | tail -10 -``` - -## Troubleshooting - -### "certificate contains no URI SAN" - -- Verify SSL passthrough is working (certificate test above) -- Ensure `useCSIDriver: true` in values -- Check SPIRE entry has synced - -### "certificate is valid for ingress.local" - -- DNS may point to wrong LoadBalancer -- Ensure Ingress TLS section has `hosts` but **no** `secretName` -- Verify NGINX has `--enable-ssl-passthrough=true` - -### ConfigMap Changes Not Taking Effect - -ConfigMaps are mounted at pod creation. Restart the deployment: - -```bash -kubectl rollout restart deployment/ -n -``` - -## Reference - -- [dir-staging](https://github.com/agntcy/dir-staging) – Example deployment with ArgoCD and SPIRE (uses `prod.*.ads.outshift.io` for the public Directory) -- [Running a Federated Directory Instance](partner-prod-federation.md) – Federation setup for connecting to the public network -- [Federation Profiles](federation-profiles.md) – Profile comparison and configuration diff --git a/docs/dir/records-validation.md b/docs/dir/records-validation.md deleted file mode 100644 index b0837cd..0000000 --- a/docs/dir/records-validation.md +++ /dev/null @@ -1,270 +0,0 @@ -# Records and Validation - -## ADS Records - -Directory uses [Open Agent Schema Framework](https://schema.oasf.outshift.com) (OASF) which defines a standardized schema for representing agents and their capabilities using [OASF Record specification](https://schema.oasf.outshift.com/objects/record). This ensures interoperability and consistency across different implementations of the directory service. - -### Content Identifier - -The content identifier of the record is a [Content IDentifier](https://github.com/multiformats/cid) (CID) hash digest which makes it: - -- Globally unique -- Content-addressable -- Collision-resistant -- Immutable - -### Verifiable Names - -Records must include a `name` field with a domain-based identifier that enables name verification. When a record uses a verifiable name: - -- The name must include a protocol prefix: `https://domain/path` or `http://domain/path`. -- The domain must host a JWKS file at `:///.well-known/jwks.json`. -- Records signed with a private key associated with a public key present in that JWKS file can be verified as authorized by the domain. - -See the [Directory CLI documentation](directory-cli.md#name-verification) for details on name verification workflows. - -### Example Email Agent - -You can generate your own example records using the [OASF Record Sample generator](https://schema.oasf.outshift.com/sample/objects/record). Below is an example OASF record for an email agent that is capable of sending and receiving emails. - -```json -{ - "schema_version": "1.0.0", - "name": "https://www.cisco.com/agents/email-agent", - "version": "v1.0.0", - "authors": ["Cisco Systems Inc."], - "description": "An agent that can send and receive emails.", - "created_at": "2025-08-11T16:20:37.159072Z", - "skills": [ - { - "id": 10306, - "name": "natural_language_processing/information_retrieval_synthesis/information_retrieval_synthesis_search" - }, - { - "id": 10202, - "name": "natural_language_processing/natural_language_generation/summarization" - }, - { - "id": 60103, - "name": "retrieval_augmented_generation/retrieval_of_information/document_retrieval" - } - ], - "locators": [ - { - "urls": [ - "https://github.com/agntcy/agentic-apps/tree/main/email_reviewer" - ], - "type": "source_code" - }, - { - "urls": [ - "https://github.com/agntcy/agentic-apps/tree/main/email_reviewer/pyproject.toml" - ], - "type": "package" - } - ] -} -``` - -!!! note - - The `name` field uses a verifiable domain-based format (`https://cisco.com/agents/email-agent`). When signed with a key authorized by the domain's JWKS file at `https://cisco.com/.well-known/jwks.json`, this record can be pulled using the convenient reference `cisco.com/agents/email-agent:v1.0.0` instead of its CID. - -## Validation - -The Directory enforces validation on all records before accepting them. -Validation is performed using the [OASF SDK](../oasf/oasf-sdk.md), which requires an OASF schema server URL for validation. - -### Configuration - -The Directory server validates records using an OASF schema URL. By default, it uses `https://schema.oasf.outshift.com`, but you can configure a different OASF instance: - -**Using environment variables:** - -```bash -# Use default OASF instance (https://schema.oasf.outshift.com) -task server:start - -# Use custom OASF instance -DIRECTORY_SERVER_OASF_API_VALIDATION_SCHEMA_URL=https://your-custom-oasf.com task server:start -``` - -**Using YAML configuration:** - -```yaml -# server.config.yml -oasf_api_validation: - schema_url: "https://schema.oasf.outshift.com" -listen_address: "0.0.0.0:8888" -``` - -!!! note - - The server itself does not have a built-in default schema URL. Deployment tools like Helm and Taskfile set `https://schema.oasf.outshift.com` as the default. When using Docker Compose or running the server binary directly, you must explicitly set the `DIRECTORY_SERVER_OASF_API_VALIDATION_SCHEMA_URL` environment variable. - -### Validation Behavior - -The Directory server uses API-based validation against the configured OASF schema server: - -- **Validation Method**: HTTP requests to the OASF schema server API. -- **Errors vs Warnings**: Only errors cause validation to fail. Warnings are returned but do not affect the validation result. -- **Schema Version Detection**: The schema version is automatically detected from each record's `schema_version` field. -- **Supported Versions**: The OASF SDK decoder supports specific schema versions (currently: 0.7.0, 0.8.0, and 1.0.0). Records using unsupported versions are not validated. -- **Unknown Classes**: Classes (such as modules, skills, and domains) not defined in the OASF schema are rejected with an error. Records must only use classes that are defined in the configured OASF schema instance. - -### OASF Instance Configurations - -While there is only one validation method (API validation), you can configure the Directory server to use different OASF instances. -The choice of OASF instance affects which records are accepted and how compatible your directory instance is with other directory instances. - -The following table shows which OASF instance configurations can exchange records with each other: - -| Instance Type | Can Pull From | Can Be Pulled By | -|--------------|---------------|------------------| -| **Official OASF Instance** | Official OASF instance only | Official OASF instance and custom instances with additional taxonomy | -| **Custom OASF Instance (Additional Taxonomy)** | Official OASF instance, custom instances with same extended taxonomy | Custom instances with same extended taxonomy only | -| **Custom OASF Instance (Changed Taxonomy)** | Custom instances with same changed taxonomy only | Custom instances with same changed taxonomy only | - -#### Official OASF Instance - -Using the official OASF instance (`https://schema.oasf.outshift.com`) provides the baseline for the official network. -Records validated here form the most strict, compatible set. - -**Configuration:** - -```yaml -oasf_api_validation: - schema_url: "https://schema.oasf.outshift.com" -``` - -#### Custom OASF Instance (Additional Taxonomy) - -Using a custom OASF instance with extensions that add to the taxonomy (with modules, skills, and domains that extend the official taxonomy but doesn't change or remove). - -Records using the extended taxonomy can only be pulled by nodes using the exact same extended taxonomy. - -**Configuration:** - -```yaml -oasf_api_validation: - schema_url: "https://your-custom-oasf-instance.com" -``` - -#### Custom OASF Instance (Changed Taxonomy) - -Using a custom OASF instance with extensions that modify the taxonomy (with modules, skills, and domains that change or remove from the official taxonomy). - -This approach is completely incompatible with all other options, can only work with nodes using the exact same changed taxonomy. - -**Configuration:** - -```yaml -oasf_api_validation: - schema_url: "https://your-custom-oasf-instance.com" -``` - -### Deploying a Local OASF Instance - -You can deploy a local OASF instance alongside the Directory server for testing or development purposes. - -#### Testing with Local OASF Server - -To test with a local OASF instance deployed alongside the directory server: - -1. Enable OASF in Helm values - - Edit `install/charts/dir/values.yaml` with the following configuration: - - ```yaml - apiserver: - oasf: - enabled: true - ``` - -2. Set schema URL to use the deployed OASF instance - - In the same file, set the schema URL to use the deployed OASF instance: - - ```yaml - apiserver: - config: - oasf_api_validation: - schema_url: "http://dir-ingress-controller.dir-server.svc.cluster.local" - ``` - - Replace `dir` with your Helm release name and `dir-server` with your namespace if different. - -3. Deploy the local OASF instance - - ```bash - task build - task deploy:local - ``` - -The OASF instance is deployed as a subchart in the same namespace and automatically configured for multi-version routing via ingress. - -#### Using a Locally Built OASF Image - -If you want to deploy with a locally built OASF image (e.g., containing `0.9.0-dev` schema files), you need to load the image into Kind before deploying. -The `task deploy:local` command automatically creates a cluster and loads images, but it doesn't load custom OASF images. - -Follow the steps below: - -1. Create the Kind cluster - - ```bash - task deploy:kubernetes:setup-cluster - ``` - - This creates the cluster and loads the Directory server images. - -2. Build your local OASF image with the `latest` tag - - ```bash - cd /path/to/oasf/server - task build - ``` - -3. Load the OASF image into Kind - - ```bash - kind load docker-image ghcr.io/agntcy/oasf-server:latest --name agntcy-cluster - ``` - -4. Configure `values.yaml` to use the local image: - - ```yaml - oasf: - enabled: true - image: - repository: ghcr.io/agntcy/oasf-server - versions: - - server: latest - schema: 0.9.0-dev - default: true - ``` - -5. Deploy the Directory - - Don't use `task deploy:local` as it will recreate the cluster. - - ```bash - task deploy:kubernetes:dir - ``` - -!!! note - - If you update the local OASF image, reload it into Kind and restart the deployment: - - ```bash - kind load docker-image ghcr.io/agntcy/oasf-server:latest --name agntcy-cluster - kubectl rollout restart deployment/dir-oasf-0-9-0-dev -n dir-server - ``` - -### Related Documentation - -For more information, see the following: - -- [OASF Validation Service](../oasf/validation.md) - Detailed validation service documentation -- [Validation Comparison](../oasf/validation-comparison.md) - Comparison between API validator and JSON Schema -- [OASF Extensions](https://github.com/agntcy/oasf/blob/main/CONTRIBUTING.md#oasf-extensions) - Information about creating OASF extensions diff --git a/docs/dir/runtime.md b/docs/dir/runtime.md deleted file mode 100644 index ce3efab..0000000 --- a/docs/dir/runtime.md +++ /dev/null @@ -1,315 +0,0 @@ -# Runtime Discovery - -Runtime Discovery is a service that watches container runtimes (Docker, Kubernetes) for workloads and provides a gRPC API for querying them. -In addition, it also resolves workloads using various resolvers (A2A, OASF) to extract and provide details about the workload's capabilities. - -## Architecture - -The system is split into two independent components: - -```mermaid -flowchart LR - subgraph Discovery["Discovery (runtime/discovery)"] - RA["Runtime Adapters
• Docker
• Kubernetes"] - RES["Resolvers
• A2A
• OASF"] - SW["Store Writer"] - end - subgraph Server["Server (runtime/server)"] - SPACE[" "] - API["gRPC API
/ListWorkloads"] - end - RA --> RES - RES --> SW - SPACE ~~~ API - Discovery ~~~ Server - SW -- write/manage --> SB["Storage Backend
• etcd (distributed)
• Kubernetes CRDs (native)"] - SB -- read/watch --> API - - style SPACE fill:none,stroke:none -``` - -## Components - -### Discovery (`runtime/discovery/`) - -The discovery component is responsible for: - -- Watching runtimes for workloads with the `org.agntcy/discover=true` label. Supported runtimes: - - - Docker: Watches Docker daemon for labeled containers. - - Kubernetes: Watches Kubernetes API for labeled pods/services. - - Extensible architecture allows adding more runtimes in the future. - -- Resolving workload metadata using configurable resolvers: - - - A2A resolver: Extracts A2A agent card from workloads with `org.agntcy/agent-type=a2a` label. - - OASF resolver: Resolves OASF records from Directory for workloads with `org.agntcy/agent-record=` label. - - Extensible architecture allows adding more resolvers in the future. - -- Writing workloads to the storage backend (etcd or CRDs). - -The storage backend can be used to expose discovered workloads to other components (e.g., clients/servers) without coupling them directly and to reduce attack surface. - -In non-Kubernetes environments, [etcd](https://etcd.io/) is recommended as the storage backend for better portability. -In Kubernetes environments, CRDs can be used for a more native experience to ensure clients can query workloads via both gRPC and the Kubernetes API. - -```mermaid -flowchart LR - n1["Runtime Discovery"] --> n3["Runtime Server (for querying workloads via gRPC)"] & n4["CRD (for querying workloads via Kubernetes API)"] - - n1@{ shape: rect} -``` - -### Server (`runtime/server/`) - -The server component provides a gRPC API for querying discovered workloads. - -## Example Setup - -### Build Container Images - -```bash -IMAGE_TAG=latest task build -``` - -### Docker Compose - -```bash -# Deploy the stack -docker compose -f runtime/install/docker/docker-compose.yml up -d - -# Deploy example workloads -docker compose -f runtime/install/examples/docker-compose.yml up -d - -# Add discovery to all networks (required for resolvers to work) -docker network connect examples_team-a runtime-discovery -docker network connect examples_team-b runtime-discovery -docker restart runtime-discovery - -# Query the API -grpcurl -plaintext localhost:8080 agntcy.dir.runtime.v1.DiscoveryService/ListWorkloads - -# Cleanup -docker compose -f runtime/install/docker/docker-compose.yml down -docker compose -f runtime/install/examples/docker-compose.yml down -``` - -### Kubernetes - -The Helm chart supports both CRD and etcd storage backends. - -#### Setup KIND Cluster - -```bash -# Create cluster -kind create cluster --name runtime - -# Load images into KIND -kind load docker-image ghcr.io/agntcy/dir-runtime-discovery:latest --name runtime -kind load docker-image ghcr.io/agntcy/dir-runtime-server:latest --name runtime -``` - -#### Deploy Example Workloads - -```bash -kubectl apply -f runtime/install/examples/k8s.workloads.yaml -``` - -#### Deploy with CRD Storage - -```bash -# Install the chart with CRD storage (default) -helm install runtime runtime/install/chart/ - -# Wait for pods -kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=discovery --timeout=60s -kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=server --timeout=60s - -# Query the gRPC API -kubectl port-forward svc/runtime-server 8080:8080 & -grpcurl -plaintext localhost:8080 agntcy.dir.runtime.v1.DiscoveryService/ListWorkloads - -# Query the Kubernetes API to see discovered workloads -kubectl get dw -``` - -#### Deploy with etcd Storage - -```bash -# Install the chart with etcd storage -helm install runtime runtime/install/chart/ \ - --set etcd.enabled=true \ - --set discovery.config.store.type=etcd \ - --set server.config.store.type=etcd - -# Wait for pods -kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=etcd --timeout=60s -kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=discovery --timeout=60s -kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=server --timeout=60s - -# Query the gRPC API -kubectl port-forward svc/runtime-server 8080:8080 & -grpcurl -plaintext localhost:8080 agntcy.dir.runtime.v1.DiscoveryService/ListWorkloads -``` - -#### Cleanup - -```bash -kind delete cluster --name runtime -``` - -## Workload Labels - -Workloads are discovered based on labels. The discovery component watches for workloads with specific labels and processes their metadata. - -### Discovery Labels - -| Label | Runtime | Description | -|-------|---------|-------------| -| `org.agntcy/discover=true` | Kubernetes | Marks a pod/service for discovery | -| `org.agntcy/discover=true` | Docker | Marks a container for discovery | - -### Resolver Labels - -Resolvers extract metadata from discovered workloads based on their labels. -They provide additional information about the workload's capabilities. - -| Label/Annotation | Description | -|------------------|-------------| -| `org.agntcy/agent-type=a2a` | Enables A2A resolver - fetches A2A agent card from workload | -| `org.agntcy/agent-record=` | Enables OASF resolver - resolves record from Directory (e.g., `my-agent:v1.0.0`) | - -To configure OASF resolver, the Directory client must be set up using environment variables (e.g., `DIRECTORY_CLIENT_SERVER_ADDRESS`, `DIRECTORY_CLIENT_AUTH_MODE`). - -### Workload Services - -Discovered workloads have a `services` field that holds metadata extracted by resolvers: - -```json -{ - "id": "4467371c-84fd-4683-ab30-93895d78bab7", - "name": "service-a2a", - "hostname": "service-a2a", - "runtime": "kubernetes", - "type": "pod", - "labels": { - "app": "service-a2a", - "org.agntcy/discover": "true", - "org.agntcy/agent-type": "a2a", - "org.agntcy/agent-record": "my-agent:1.0.0" - }, - "addresses": [ - "10-244-0-9.team-a.pod" - ], - "ports": [ - "8080", - "9999" - ], - "isolationGroups": [ - "team-a" - ], - "services": { - "a2a": { - "name": "My Agent", - "description": "...", - "capabilities": [...] - }, - "oasf": { - "cid": "baf123", - "name": "my-agent:1.0.0", - "record": { - "name": "my-agent", - "version": "1.0.0", - "skills": [...] - } - } - } -} -``` - -## Configuration - -### Discovery Component - -| Environment Variable | Description | Default | -|---------------------|-------------|---------| -| `DISCOVERY_WORKERS` | Number of resolver workers | `16` | -| `DISCOVERY_STORE_TYPE` | Storage type (`etcd`, `crd`) | `etcd` | -| `DISCOVERY_STORE_ETCD_HOST` | etcd server hostname | `localhost` | -| `DISCOVERY_STORE_ETCD_PORT` | etcd server port | `2379` | -| `DISCOVERY_STORE_ETCD_USERNAME` | etcd username for authentication | `` | -| `DISCOVERY_STORE_ETCD_PASSWORD` | etcd password for authentication | `` | -| `DISCOVERY_STORE_ETCD_DIAL_TIMEOUT` | Timeout for connecting to etcd | `5s` | -| `DISCOVERY_STORE_ETCD_WORKLOADS_PREFIX` | etcd key prefix for workloads | `/discovery/workloads/` | -| `DISCOVERY_STORE_CRD_NAMESPACE` | Namespace to store workloads in | `default` | -| `DISCOVERY_STORE_CRD_KUBECONFIG` | Path to kubeconfig file (empty for in-cluster) | `` | -| `DISCOVERY_STORE_CRD_RESYNC_PERIOD` | How often to resync the cache from the API server | `30s` | -| `DISCOVERY_RUNTIME_TYPE` | Runtime type (`docker`, `kubernetes`) | `docker` | -| `DISCOVERY_RUNTIME_DOCKER_HOST` | Docker daemon socket path | `unix:///var/run/docker.sock` | -| `DISCOVERY_RUNTIME_DOCKER_LABEL_KEY` | Label key to filter containers | `org.agntcy/discover` | -| `DISCOVERY_RUNTIME_DOCKER_LABEL_VALUE` | Label value to filter containers | `true` | -| `DISCOVERY_RUNTIME_KUBERNETES_KUBECONFIG` | Path to kubeconfig file (empty for in-cluster) | `` | -| `DISCOVERY_RUNTIME_KUBERNETES_NAMESPACE` | Namespace to watch (empty for all namespaces) | `` | -| `DISCOVERY_RUNTIME_KUBERNETES_LABEL_KEY` | Label key to filter pods | `org.agntcy/discover` | -| `DISCOVERY_RUNTIME_KUBERNETES_LABEL_VALUE` | Label value to filter pods | `true` | -| `DISCOVERY_RESOLVER_A2A_ENABLED` | Enable A2A resolver | `true` | -| `DISCOVERY_RESOLVER_A2A_TIMEOUT` | Timeout for A2A discovery | `5s` | -| `DISCOVERY_RESOLVER_A2A_PATHS` | Comma-separated list of paths to probe for A2A discovery | `/.well-known/agent-card.json,/.well-known/card.json` | -| `DISCOVERY_RESOLVER_A2A_LABEL_KEY` | Label key to identify A2A workloads | `org.agntcy/agent-type` | -| `DISCOVERY_RESOLVER_A2A_LABEL_VALUE` | Label value to identify A2A workloads | `a2a` | -| `DISCOVERY_RESOLVER_OASF_ENABLED` | Enable OASF resolver | `true` | -| `DISCOVERY_RESOLVER_OASF_TIMEOUT` | Timeout for OASF resolution | `5s` | -| `DISCOVERY_RESOLVER_OASF_LABEL_KEY` | Label key to identify OASF workloads | `org.agntcy/agent-record` | - -#### Discovery OASF Resolver - -The OASF resolver requires Directory client configuration via environment variables. -These are the same as those used by the Directory client library, e.g. `DIRECTORY_CLIENT_SERVER_ADDRESS`. -Refer to the [Directory Go SDK](./directory-sdk.md#go-sdk) for all available options. - -When a workload has the configured OASF resolver label, the resolver attempts to fetch the corresponding record from Directory and validate its signature before attaching it to the workload. - -### Server Component - -| Environment Variable | Description | Default | -|---------------------|-------------|---------| -| `SERVER_HOST` | Server bind address | `0.0.0.0` | -| `SERVER_PORT` | Server listen port | `8080` | -| `SERVER_STORE_TYPE` | Storage type (`etcd`, `crd`) | `etcd` | -| `SERVER_STORE_ETCD_HOST` | etcd server hostname | `localhost` | -| `SERVER_STORE_ETCD_PORT` | etcd server port | `2379` | -| `SERVER_STORE_ETCD_USERNAME` | etcd username for authentication | `` | -| `SERVER_STORE_ETCD_PASSWORD` | etcd password for authentication | `` | -| `SERVER_STORE_ETCD_DIAL_TIMEOUT` | Timeout for connecting to etcd | `5s` | -| `SERVER_STORE_ETCD_WORKLOADS_PREFIX` | etcd key prefix for workloads | `/discovery/workloads/` | -| `SERVER_STORE_CRD_NAMESPACE` | Namespace to read workloads from | `default` | -| `SERVER_STORE_CRD_KUBECONFIG` | Path to kubeconfig file (empty for in-cluster) | `` | -| `SERVER_STORE_CRD_RESYNC_PERIOD` | How often to resync the cache from the API server | `30s` | - - -## gRPC API - -The server exposes a gRPC API defined in `proto/agntcy/dir/runtime/v1/discovery_service.proto`. - -### GetWorkload - -Get a specific workload by ID, name, or hostname. - -```bash -grpcurl -plaintext -d '{"id": "my-service"}' \ - localhost:8080 agntcy.dir.runtime.v1.DiscoveryService/GetWorkload -``` - -### ListWorkloads - -Stream all workloads with optional label filters. Labels support regex patterns. - -```bash -# List all workloads -grpcurl -plaintext -d '{}' \ - localhost:8080 agntcy.dir.runtime.v1.DiscoveryService/ListWorkloads - -# Filter by labels (supports regex) -grpcurl -plaintext -d '{"labels": {"org.agntcy/agent-type": "a2a"}}' \ - localhost:8080 agntcy.dir.runtime.v1.DiscoveryService/ListWorkloads -``` diff --git a/docs/dir/scenarios.md b/docs/dir/scenarios.md deleted file mode 100644 index df6d06b..0000000 --- a/docs/dir/scenarios.md +++ /dev/null @@ -1,661 +0,0 @@ -# Features and Usage Scenarios - -This document defines a basic overview of main Directory features, components, and usage -scenarios. All code snippets below are tested against the Directory `v1.0.0` release. - -!!! note - Although the following example is shown for a CLI-based usage scenario, the same - functionality can be performed using language-specific SDKs. - -The Agent Directory Service is also accessible through the Directory MCP Server. It provides a standardized interface for AI assistants and tools to interact with the Directory system and work with OASF agent records. See the [MCP Server documentation](directory-mcp.md) for more information. - -## Prerequisites - -The following prerequisites are required to follow the examples below: - -- Locally available Directory CLI client -- Running instance of Directory API server - -To deploy the necessary components, please refer to the [Getting Started](getting-started.md) -guide. - -## Build - -This example demonstrates how to define a Record using provided tooling to prepare for -publication. - -To start, generate an example Record that matches the data model schema defined in -[OASF Record specification](https://schema.oasf.outshift.com/objects/record) using the -[OASF Record Sample generator](https://schema.oasf.outshift.com/sample/objects/record). - -```bash -# Generate an example data model -cat << EOF > record.json -{ - "name": "https://example.com/agents/my-record", - "version": "v1.0.0", - "description": "insert description here", - "schema_version": "1.0.0", - "skills": [ - { - "id": 201, - "name": "images_computer_vision/image_segmentation" - } - ], - "authors": [ - "Jane Doe" - ], - "created_at": "2025-08-11T16:20:37.159072Z", - "locators": [ - { - "type": "source_code", - "urls": [ "https://github.com/agntcy/oasf/blob/main/record" ] - } - ] -} -EOF -``` - -## Store - -This example demonstrates the interaction with the storage layer using the CLI client. -The storage layer uses an OCI-compliant registry to store records as OCI artifacts with -[content-addressable identifiers](https://github.com/multiformats/cid) (CIDs). When a record -is pushed, it is stored as an OCI blob and the CID is calculated by converting the SHA256 -OCI digest into a CIDv1 format using CID multihash encoding. Each record is then tagged with -its CID in the registry, enabling direct lookup and ensuring content integrity through -cryptographic addressing. - -### Supported Registries - -The Directory supports multiple OCI-compatible registry backends: - -| Registry Type | Description | -|---------------|-------------| -| `zot` | [Zot](https://github.com/project-zot/zot) OCI registry (default) | -| `ghcr` | GitHub Container Registry | -| `dockerhub` | Docker Hub | - -#### Registry Configuration - -The registry backend is configured via environment variables on the Directory server: - -| Environment Variable | Description | Default | -|---------------------|-------------|---------| -| `DIRECTORY_SERVER_STORE_OCI_TYPE` | Registry type (`zot`, `ghcr`, `dockerhub`) | `zot` | -| `DIRECTORY_SERVER_STORE_OCI_REGISTRY_ADDRESS` | Registry address | `127.0.0.1:5000` | -| `DIRECTORY_SERVER_STORE_OCI_REPOSITORY_NAME` | Repository name | `dir` | - -### Authentication Configuration - -Credentials for the registry are configured via environment variables: - -| Environment Variable | Description | -|---------------------|-------------| -| `DIRECTORY_SERVER_STORE_OCI_AUTH_CONFIG_USERNAME` | Username for basic authentication | -| `DIRECTORY_SERVER_STORE_OCI_AUTH_CONFIG_PASSWORD` | Password for basic authentication | -| `DIRECTORY_SERVER_STORE_OCI_AUTH_CONFIG_ACCESS_TOKEN` | Access token for token-based authentication | -| `DIRECTORY_SERVER_STORE_OCI_AUTH_CONFIG_INSECURE` | Skip TLS verification (default: `true`) | - -### Configuration Examples - -**Zot (Local Development)** - -```bash -export DIRECTORY_SERVER_STORE_OCI_TYPE=zot -export DIRECTORY_SERVER_STORE_OCI_REGISTRY_ADDRESS=localhost:5000 -export DIRECTORY_SERVER_STORE_OCI_REPOSITORY_NAME=dir -export DIRECTORY_SERVER_STORE_OCI_AUTH_CONFIG_INSECURE=true -``` - -**GitHub Container Registry (GHCR)** - -```bash -export DIRECTORY_SERVER_STORE_OCI_TYPE=ghcr -export DIRECTORY_SERVER_STORE_OCI_REGISTRY_ADDRESS=ghcr.io -export DIRECTORY_SERVER_STORE_OCI_REPOSITORY_NAME=your-org/dir -export DIRECTORY_SERVER_STORE_OCI_AUTH_CONFIG_USERNAME=your-github-username -export DIRECTORY_SERVER_STORE_OCI_AUTH_CONFIG_PASSWORD=your-github-token -export DIRECTORY_SERVER_STORE_OCI_AUTH_CONFIG_INSECURE=false -``` - -!!! warning - GHCR does not support record deletion via the OCI API. Attempting to - delete a record when using GHCR will return an error. - To manage packages hosted on GHCR, use the GitHub UI, REST API, or - GraphQL API instead. See - [Deleting and restoring a package](https://docs.github.com/en/packages/learn-github-packages/deleting-and-restoring-a-package) - for details. - -**Docker Hub** - -```bash -export DIRECTORY_SERVER_STORE_OCI_TYPE=dockerhub -export DIRECTORY_SERVER_STORE_OCI_REGISTRY_ADDRESS=docker.io -export DIRECTORY_SERVER_STORE_OCI_REPOSITORY_NAME=your-username/dir -export DIRECTORY_SERVER_STORE_OCI_AUTH_CONFIG_USERNAME=your-dockerhub-username -export DIRECTORY_SERVER_STORE_OCI_AUTH_CONFIG_PASSWORD=your-dockerhub-token -export DIRECTORY_SERVER_STORE_OCI_AUTH_CONFIG_INSECURE=false -``` - -### Basic Operations - -Once the server is configured, the CLI operations work the same regardless of the underlying -registry backend: - -```bash -# Push the record and store its CID to a file -dirctl push record.json > record.cid - -# Set the CID as a variable for easier reference -RECORD_CID=$(cat record.cid) - -# Pull the record by CID -# Returns the same data as record.json -dirctl pull $RECORD_CID - -# Pull the record by name (if it has a verifiable name) -# Returns the same data as record.json -dirctl pull example.com/agents/my-record:v1.0.0 - -# Lookup basic metadata about the record by CID -# Returns annotations, creation timestamp and OASF schema version -dirctl info $RECORD_CID - -# Lookup basic metadata by name -dirctl info example.com/agents/my-record:v1.0.0 -``` - -Records with verifiable names can be referenced using Docker-style formats: - -- `example.com/agents/my-record` - Latest version -- `example.com/agents/my-record:v1.0.0` - Specific version -- `example.com/agents/my-record:v1.0.0@bafyreib...` - Hash-verified lookup - -Name-based references work with `pull`, `info`, and `naming verify` commands. - -## Signing and Verification - -Establishing trust and authenticity is critical in distributed AI agent ecosystems, where -records may be shared across multiple nodes and networks. By cryptographically signing -records, publishers can prove authorship and ensure data integrity, while consumers can -verify that records haven't been tampered with and originate from trusted sources before -deploying or executing agent code. - -Signatures and public keys are stored in the OCI registry as referrer artifacts that -maintain subject relationships with their associated records. When a record is signed, the -signature is attached as a Cosign-compatible OCI artifact. Public keys are similarly stored -as separate OCI artifacts, creating a verifiable chain of trust through OCI's native -referrer mechanism. - -Server-side verification leverages Zot's trust extension through GraphQL queries that check -both signature validity and trust status. When public keys are uploaded to Zot, they enable -the registry to mark signatures as "trusted" when they can be cryptographically verified -against the stored public keys. The verification process queries Zot's search API to -retrieve signature metadata including the `IsSigned` and `IsTrusted` status, allowing the -Directory server to make trust decisions based on the cryptographic verification performed -by the underlying OCI registry infrastructure. - -### Method 1: OIDC-based Interactive - -This process relies on creating and uploading to the OCI registry a signature for the record -using identity-based OIDC signing flow which can later be verified. The signing process -opens a browser window to authenticate the user with an OIDC identity provider. These -operations are implemented using [Sigstore](https://www.sigstore.dev/). - -```bash -# Push record with signature -dirctl push record.json --sign - -# Alternatively, sign a pushed record -dirctl sign $RECORD_CID - -# Verify record -dirctl verify $RECORD_CID -``` - -### Method 2: OIDC-based Non-Interactive - -This method is designed for automated environments such as CI/CD pipelines where -browser-based authentication is not available. It uses OIDC tokens provided by the -execution environment (like GitHub Actions) to sign records. The signing process uses a -pre-obtained OIDC token along with provider-specific configuration to establish identity -without user interaction. - -``` -- name: Push and sign record - run: | - bin/dirctl push record.json --sign \ - --oidc-token ${{ steps.oidc-token.outputs.token }} \ - --oidc-provider-url "https://token.actions.githubusercontent.com" \ - --oidc-client-id "https://github.com/${{ github.repository }}/.github/workflows/demo.yaml@${{ github.ref }}" - -- name: Run verify command - run: | - echo "Running dir verify command" - bin/dirctl verify $RECORD_CID -``` - -### Method 3: Self-Managed Keys - -This method is suitable for non-interactive use cases, such as CI/CD pipelines, where -browser-based authentication is not possible or desired. Instead of OIDC, a signing keypair -is generated (e.g., with Cosign), and the private key is used to sign the record. - -```bash -# Generate a key-pair for signing -# This creates 'cosign.key' (private) and 'cosign.pub' (public) -cosign generate-key-pair - -# Set COSIGN_PASSWORD shell variable if you password-protected the private key -export COSIGN_PASSWORD=your_password_here - -# Push record with signature -dirctl push record.json --sign --key cosign.key - -# Verify the signed record -dirctl verify $RECORD_CID -``` - -## Name Verification - -Name verification proves that the signing key is authorized by the domain claimed in the -record's name field. This provides cryptographic proof of domain ownership and enables -human-readable references while maintaining security. - -### Requirements - -To use name verification, your record must meet these requirements: - -- Record name must include a protocol prefix: `https://domain/path` or `http://domain/path` -- A [JWKS (JSON Web Key Set)](https://datatracker.ietf.org/doc/html/rfc7517) file must be hosted at `:///.well-known/jwks.json` -- The record must be signed with the private key corresponding to a public key present in that JWKS file - -### Workflow - -```bash -# 1. Create a record with a verifiable name (already done in Build section) -# The record.json has: "name": "https://example.com/agents/my-record" - -# 2. Ensure your domain hosts a JWKS file -# Example: https://example.com/.well-known/jwks.json -# This file should contain the public key corresponding to your signing key - -# 3. Push the record -RECORD_CID=$(dirctl push record.json --output raw) -echo "Stored with CID: $RECORD_CID" - -# 4. Sign the record (triggers automatic verification) -dirctl sign $RECORD_CID --key cosign.key - -# 5. Verify the name authorization -# By CID -dirctl naming verify $RECORD_CID --output json - -# By name (latest version) -dirctl naming verify example.com/agents/my-record --output json - -# By name with specific version -dirctl naming verify example.com/agents/my-record:v1.0.0 --output json -``` - -### Verification Response - -When verification succeeds, you'll receive a response like: - -```json -{ - "cid": "bafyreib...", - "verified": true, - "domain": "example.com", - "method": "jwks", - "key_id": "key-1", - "verified_at": "2026-01-21T10:30:00Z" -} -``` - -### Using Verified Names - -Once verified, you can use convenient name-based references instead of CIDs: - -```bash -# Pull by name (latest version) -dirctl pull example.com/agents/my-record - -# Pull specific version -dirctl pull example.com/agents/my-record:v1.0.0 - -# Pull with hash verification (fails if CID doesn't match) -dirctl pull example.com/agents/my-record:v1.0.0@$RECORD_CID - -# Get info by name -dirctl info example.com/agents/my-record:v1.0.0 --output json -``` - -### Version Resolution - -When no version is specified, commands return the most recently created record (by the -record's `created_at` field). This allows non-semver tags like `latest`, `dev`, or `stable`: - -```bash -# These all pull the most recent version -dirctl pull example.com/agents/my-record -dirctl pull example.com/agents/my-record:latest -dirctl pull example.com/agents/my-record:dev -``` - -## Announce - -This example demonstrates how to publish records to allow content discovery across the -network. Publication requests are processed asynchronously in the background using a -scheduler that manages DHT announcements. To avoid stale data, it is recommended to -republish the data periodically as the data across the network has TTL. - -Note that this operation only works for the objects already pushed to the local storage -layer, i.e., it is required to first push the data before publication. - -```bash -# Publish the record across the network -dirctl routing publish $RECORD_CID -``` - -If the data is not published to the network, it cannot be discovered by other peers. For -published data, peers may try to reach out over the network to request specific objects for -verification and replication. Network publication may fail if you are not connected to the -network. - -## Discover - -This example demonstrates how to discover records both locally and across the network using -two distinct commands for different use cases. - -### Local Discovery - -Use `dirctl routing list` to discover records stored locally on this peer only. This queries -the server's local storage index and does not search other peers on the network. - -```bash -# List all local records -dirctl routing list - -# List local records with specific skill -dirctl routing list --skill "images_computer_vision/image_segmentation" - -# List records with multiple criteria (AND logic) -dirctl routing list --skill "images_computer_vision/image_segmentation" \ - --locator "source_code" - -# List specific record by CID -dirctl routing list --cid $RECORD_CID -``` - -### Network Discovery - -Use `dirctl routing search` to discover records from other peers across the network. This -uses cached network announcements and filters out local records. - -```bash -# Search for records with exact skill match -dirctl routing search --skill "images_computer_vision/image_segmentation" - -# Search for records with skill prefix match (finds all NLP-related skills) -dirctl routing search --skill "images_computer_vision" - -# Search with multiple criteria (OR logic with minimum score) -dirctl routing search --skill "images_computer_vision" \ - --skill "audio" \ - --min-score 2 - -# Search with result limiting -dirctl routing search --skill "images_computer_vision" \ - --limit 5 -``` - -Network search supports hierarchical matching where skills, domains, and modules use both -exact and prefix matching (e.g., `images_computer_vision` matches both `images_computer_vision` -and `images_computer_vision/image_segmentation` as a prefix). - -Note that network search results are not guaranteed to be available, valid, or up to date as -they rely on cached announcements from other peers. - -## Search - -This example demonstrates how to search for records in your local directory using various filters -and query parameters. The search functionality allows you to find records based on specific -attributes like name, version, skills, locators, domains and modules using structured -filters with wildcard support. All searches are case insensitive. - -Search operations leverage an SQLite database for efficient record indexing and querying, -supporting pagination and returning Content Identifier (CID) values that can be used with -other Directory commands like `pull`, `info`, and `verify`. - -```bash -# Basic search for records by name -dirctl search --name "my-agent-name" - -# Search for records with verifiable domain-based names -dirctl search --name "example.com/agents/my-record" - -# Search for records with a specific version -dirctl search --version "v1.0.0" - -# Search for records that have a particular skill by ID -dirctl search --skill-id "10201" - -# Search for records with a specific skill name -dirctl search --skill "images_computer_vision/image_segmentation" - -# Search for records with a specific locator type -dirctl search --locator "docker-image" - -# Search for records with a specific domain -dirctl search --domain "healthcare" - -# Search for records with a specific module -dirctl search --module "runtime/framework" - -# Combine multiple filters (AND operation) -dirctl search \ - --name "my-agent" \ - --version "v1.0.0" \ - --skill "images_computer_vision/image_segmentation" - -# Use multiple values for the same filter (OR operation within filter type) -dirctl search \ - --skill "images_computer_vision" \ - --skill "natural_language_processing" - -# Use pagination to limit results and specify offset -dirctl search \ - --skill "images_computer_vision/image_segmentation" \ - --limit 10 \ - --offset 0 - -# Get the next page of results -dirctl search \ - --skill "images_computer_vision/image_segmentation" \ - --limit 10 \ - --offset 10 -``` - -### Wildcard Search - -The search functionality supports wildcard patterns for flexible matching: - -```bash -# Asterisk (*) wildcard - matches zero or more characters -dirctl search --name "web*" # Find all web-related agents -dirctl search --name "example.com/*" # Find all agents from example.com domain -dirctl search --version "v1.*" # Find all v1.x versions -dirctl search --skill "audio*" # Find Audio-related skills -dirctl search --locator "http*" # Find HTTP-based locators - -# Question mark (?) wildcard - matches exactly one character -dirctl search --version "v1.0.?" # Find version v1.0.x (single digit) -dirctl search --name "???api" # Find 3-character names ending in "api" -dirctl search --skill "Pytho?" # Find skills with single character variations - -# Complex wildcard combinations -dirctl search --name "api-*-service" --version "v2.*" -dirctl search --skill "*machine*learning*" -``` - -**Available Search Flags:** - -- `--name ` - Search by record name -- `--version ` - Search by record version -- `--skill-id ` - Search by skill ID number -- `--skill ` - Search by skill name -- `--locator ` - Search by locator type -- `--domain ` - Search by domain -- `--module ` - Search by module name - -**Search Logic:** - -Multiple flags of different types are combined with AND logic (all criteria must match). -Multiple flags of the same type are combined with OR logic (any criteria can match). -For example, `--skill "audio" --skill "video" --locator "docker-image"` finds records that have -either "audio" OR "video" skills AND use "docker-image" locators. - -## Sync - -The sync feature enables one-way synchronization of records and other objects between -remote Directory instances and your local node. This feature supports distributed AI agent -ecosystems by allowing you to replicate content from multiple remote directories, creating -local mirrors for offline access, backup, and cross-network collaboration. - -**How Sync Works**: Directory uses [regsync](https://github.com/regclient/regclient/tree/main/cmd/regsync) -(from regclient) as the synchronization engine for all registry types. When you create a sync -operation, the reconciler generates a regsync configuration and runs `regsync once` to pull -content from remote registries. Objects are stored as OCI artifacts (manifests, blobs, and -tags), enabling container-native synchronization with secure credential exchange between -Directory nodes. - -This example demonstrates how to synchronize records between remote directories and your -local instance. - -### Basic Sync Operations - -```bash -# Create a sync operation (reconciler will run sync and pull all records from remote) -dirctl sync create https://remote-directory.example.com:8888 - -# Sync specific records by CID -dirctl sync create https://remote-directory.example.com:8888 \ - --cids cid1,cid2,cid3 - -# List all sync operations -dirctl sync list - -# Check the status of a specific sync operation -dirctl sync status - -# Mark a sync for deletion (reconciler will process and remove it) -dirctl sync delete -``` - -### Advanced Sync with Routing - -You can combine routing search with sync operations to selectively synchronize records that -match specific criteria: - -```bash -# Search for agents with a given skill across -# the network and sync them automatically -dirctl routing search --skill "Audio" --output json | dirctl sync create --stdin -``` - -This creates separate sync operations for each remote peer found in the search results, -syncing only the specific CIDs that matched your search criteria. - -## Import - -The import feature extends Directory's synchronization capabilities beyond DIR-to-DIR sync to support heterogeneous external registries. This enables you to aggregate agent records from multiple registry types into your local Directory instance. - -**How Import Works**: The import system uses registry-specific adapters to fetch records from external sources and transform them into OASF-compliant records. Each registry type has its own import logic that handles authentication, pagination, filtering, and data transformation. Records are automatically deduplicated and can be enriched with LLM-powered skill and domain mapping to ensure consistency with the OASF schema. - -**How Translation and Enrichment Work**: Records are transformed from external registry data to OASF-compliant format, directly impacting how records are indexed and discovered across the network. - -Three methods are available: - -- **Basic translation** uses [OASF-SDK basic translation](../oasf/translation.md) with rule-based mapping. This method is fast and deterministic but produces a record without any skills or domains, requiring manual or LLM-based enrichment after the initial translation. -- **Local LLM enrichment** runs LLM locally for intelligent skill and domain mapping, requiring local LLM runtime. -- **Remote LLM enrichment** uses external LLM services for skill and domain mapping, requiring API credentials. Both LLM methods require [MCPHost environment setup](https://github.com/mark3labs/mcphost?tab=readme-ov-file#environment-setup). - -This example demonstrates how to import records from external registries into your local Directory instance. The import feature supports automated batch imports with filtering, deduplication, and optional LLM-based enrichment. - -**Supported Registries:** - -- `mcp` - [Model Context Protocol registry v0.1](https://github.com/modelcontextprotocol/registry) - -### Basic Usage - -```bash -# Import from MCP registry -dirctl import --type=mcp --url=https://registry.modelcontextprotocol.io/v0.1 -``` - -### Automated Imports - -For Kubernetes deployments, you can configure automated imports using the [Helm chart configuration](https://github.com/agntcy/dir/blob/2aea0d670ef9d537b9a9237928dd1af7b02de447/install/charts/dirctl/values.yaml#L55): - -```yaml -cronjobs: - # Import cronjob - sync from MCP registry every 6 hours - import-mcp: - enabled: true - schedule: '0 */6 * * *' # Every 6 hours - args: - - 'import' - - '--type=mcp' - - '--url=https://registry.modelcontextprotocol.io/v0.1' -``` - -### Common Import Options - -```bash -# Basic import from MCP registry -dirctl import --type=mcp --url=https://registry.modelcontextprotocol.io/v0.1 - -# Import with filtering and limits -dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --filter=version=latest \ - --limit=50 - -# Import with LLM enrichment -dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --enrich - -# Force reimport of existing records (bypasses deduplication) -dirctl import --type=mcp \ - --url=https://registry.modelcontextprotocol.io/v0.1 \ - --force -``` - -For comprehensive documentation including all configuration options, filtering capabilities, LLM enrichment setup, and advanced usage examples, see the [CLI Import Workflow documentation](https://github.com/agntcy/dir/tree/main/cli#-import-workflow). - -## gRPC Error Codes - -The following table lists the gRPC error codes returned by the server APIs, along with a -description of when each code is used: - -| Error Code | Description | -| -------------------------- | ------------------------------------------------------------ | -| `codes.InvalidArgument` | When client provides an invalid or malformed argument, such | -| | as a missing or invalid record reference or record. | -| `codes.NotFound` | When the requested object does not exist in the local store | -| | or across the network. | -| `codes.FailedPrecondition` | When the server environment or configuration is not in the | -| | required state (e.g., failed to create a directory or temp | -| | file). | -| `codes.Internal` | When an unexpected internal error occurs, such as I/O | -| | failures, serialization errors, or other server-side | -| | issues. | -| `codes.Canceled` | When the operation is canceled by the client or context | -| | expires. | -| `codes.Unauthenticated` | When the client is not authenticated to perform the | -| | operation. | -| `codes.PermissionDenied` | When the client does not have permission to perform the | -| | operation. | diff --git a/docs/dir/trust-model.md b/docs/dir/trust-model.md deleted file mode 100644 index b7cde23..0000000 --- a/docs/dir/trust-model.md +++ /dev/null @@ -1,291 +0,0 @@ -# Security Trust Model - -## Overview - -Directory is a system designed to provide secure, authenticated, and authorized access to -services and resources across multiple environments and organizations. It leverages -[SPIRE](https://spiffe.io/) to manage workload identities and -enable zero-trust security principles. - -[SPIRE](https://spiffe.io/docs/latest/spire-about/) (SPIFFE Runtime Environment) is an open-source system that provides automated, -cryptographically secure identities to workloads in modern infrastructure. It implements the -[SPIFFE](https://spiffe.io/) (Secure Production Identity Framework For Everyone) standard, enabling zero-trust -security by assigning each workload a unique, verifiable identity (SVID). - -In the Directory project, SPIRE is used to: - -- Securely identify and authenticate workloads (services, applications, etc.). -- Enable mutual transport layer security (mTLS) between services. -- Support dynamic, scalable, and multi-environment deployments. -- Enable interconnectivity between different organizations. -- Provide primitives for authorization logic. - -## Authentication and Authorization - -### Authentication - -SPIRE provides strong, cryptographically verifiable identities (SPIFFE IDs) to every -workload. These identities are used for: - -- **Workload Authentication:** Every service, whether running in Kubernetes, on a VM, or on bare metal, receives a unique SPIFFE ID (e.g., `spiffe://dir.example/ns/default/sa/my-service`). -- **Cross-Organization Authentication:** Through federation, workloads from different organizations or clusters can mutually authenticate using their SPIFFE IDs, without the need to implement custom cross-org authentication logic. -- **Mutual TLS (mTLS):** SPIRE issues SVIDs (X.509 certificates) that are used to establish mTLS connections, ensuring both parties are authenticated and communication is encrypted. - -**What problem does SPIRE solve?** - -- Eliminates the need to build and maintain custom authentication systems for each environment or organization. -- Provides a standard, interoperable identity for every workload, regardless of where it runs. -- Enables secure, automated trust establishment between independent organizations or clusters. - -#### How Directory uses SPIRE for Authentication - -- **Workload Identity:** Each Directory component (API server, clients, etc.) is assigned a SPIFFE ID based on its SPIRE Agent configuration. -- **Cross-Organization Authentication:** Directory can authenticate workloads from other organizations or clusters using their SPIFFE IDs, enabling secure communication without custom integration. -- **Secure Communication:** Directory can establish mTLS connections between components using the SVIDs issued by SPIRE, ensuring secure and authenticated communication. - -### Authorization - -SPIRE itself does not enforce authorization, but it enables fine-grained authorization by providing strong workload identities: - -- **Policy-Based Access Control:** Applications and infrastructure can use SPIFFE IDs to define and enforce access policies (e.g., only workloads with a specific SPIFFE ID can access a sensitive API). -- **Attribute-Based Authorization:** SPIFFE IDs can encode attributes (namespace, service account, environment) that can be used in authorization decisions. -- **Cross-Domain Authorization:** Because SPIRE federates trust domains, authorization policies can include or exclude identities from other organizations or clusters, enabling secure collaboration without manual certificate management. - -**What problem does SPIRE solve?** - -- Enables authorization decisions based on workload identity, not just network location or static credentials. -- Simplifies policy management by using a standard identity format (SPIFFE ID) across all environments. -- Makes it possible to securely authorize workloads from federated domains (e.g., partner organizations, multi-cloud, hybrid setups) without custom integration. - -#### How Directory uses SPIRE for Authorization - -- **Policy Enforcement:** Directory components can enforce access control policies based on the SPIFFE IDs of incoming requests, ensuring that only authorized workloads can access specific services or APIs. -- **Access Control:** Directory can leverage attributes encoded in SPIFFE IDs to implement fine-grained access control policies. -- **Federated Authorization:** Directory can use SPIFFE IDs to authorize workloads from other organizations or clusters, enabling secure collaboration without custom integration. - -Currently, Directory implements static authorization policies based on SPIFFE IDs, with plans to enhance this with dynamic, attribute-based policies in future releases. The Authorization policies are enforced based on external trust domains in the following manner: - -| API Method | Authorized Trust Domains | -| --------------------------------- | ------------------------------------------- | -| `*` | Your own trust domain (e.g., `dir.example`) | -| `Store.Pull` | External Trust domain | -| `Store.Lookup` | External Trust domain | -| `Store.PullReferrer` | External Trust domain | -| `Sync.RequestRegistryCredentials` | External Trust domain | - -## Topology - -The Directory's security trust schema supports both single and federated trust domain topology setup, with SPIRE deployed across various environments: - -### Single Trust Domain - -- **SPIRE Server**: Central authority for the trust domain. - -- **SPIRE Agents**: Deployed in different environments, connect to the SPIRE Server: - - - Kubernetes clusters (as DaemonSets or sidecars) - - VMs (as systemd services or processes) - - Bare metal - -- **Workloads**: Obtain identities from local SPIRE Agent via the Workload API. - -```mermaid -flowchart LR - subgraph Trust_Domain[Trust Domain: example.org] - SPIRE_SERVER[SPIRE Server] - AGENT_K8S1[SPIRE Agent K8s] - AGENT_VM[SPIRE Agent VM] - AGENT_SSH[SPIRE Agent Bare Metal] - SPIRE_SERVER <--> AGENT_K8S1 - SPIRE_SERVER <--> AGENT_VM - SPIRE_SERVER <--> AGENT_SSH - end -``` - -### Federated Trust Domains - -- Each environment (e.g., cluster, organization) runs its own SPIRE Server and Agents -- SPIRE Servers exchange bundles to establish federation -- Enables secure, authenticated communication between workloads in different domains - -For step-by-step federation setup with the public Directory network, see [Running a Federated Directory Instance](partner-prod-federation.md). For technical details on federation profiles (https_web vs https_spiffe), see [Federation Profiles](federation-profiles.md). - -```mermaid -flowchart TD - subgraph DIR_Trust_Domain[Trust Domain: dir.example] - DIR_SPIRE_SERVER[SPIRE Server] - DIR_SPIRE_AGENT1[SPIRE Agent K8s] - DIR_SPIRE_AGENT1[SPIRE Agent VM] - DIR_SPIRE_SERVER <--> DIR_SPIRE_AGENT1 - DIR_SPIRE_SERVER <--> DIR_SPIRE_AGENT2 - end - subgraph DIRCTL_Trust_Domain[Trust Domain: dirctl.example] - DIRCTL_SPIRE_SERVER[SPIRE Server] - DIRCTL_SPIRE_AGENT1[SPIRE Agent k8s] - DIRCTL_SPIRE_AGENT2[SPIRE Agent VM] - DIRCTL_SPIRE_SERVER <--> DIRCTL_SPIRE_AGENT1 - DIRCTL_SPIRE_SERVER <--> DIRCTL_SPIRE_AGENT2 - end - DIR_SPIRE_SERVER <-.->|"Federation (SPIFFE Bundle)"| DIRCTL_SPIRE_SERVER -``` - -## Deployment - -In order to deploy Directory with Security Trust Model support, it is required -to deploy SPIRE components. - -### SPIRE Server - -The SPIRE Server is configured as follows: - -- **Deployment Options**: Can be deployed either as a Kubernetes or -as a standalone service, providing flexibility for different infrastructure setups. - -- **Trust Domain Configuration**: Requires a unique trust domain name -(such as dir.example) to establish its identity scope. - -- **Federation Support**: Federation is enabled to allow cross-domain -trust relationships between different SPIRE deployments. -If the federation is not required, it can be left disabled. -See [Running a Federated Directory Instance](partner-prod-federation.md) for configuration. - -- **Bundle Endpoint**: Exposes a bundle endpoint that enables federation -by allowing other SPIRE servers to exchange trust bundles. -See [Federation Profiles](federation-profiles.md) for profile options (https_web, https_spiffe). - -```bash -# Set the trust domain -export TRUST_DOMAIN="my-service.local" - -# Add the SPIFFE Helm chart repository -helm repo add spiffe https://spiffe.github.io/helm-charts-hardened - -# Install SPIRE CRDs -helm upgrade spire-crds spire-crds \ - --repo https://spiffe.github.io/helm-charts-hardened/ \ - --create-namespace -n spire-crds \ - --install \ - --wait \ - --wait-for-jobs \ - --timeout "15m" - -# Install SPIRE Server with federation enabled -helm upgrade spire spire \ - --repo https://spiffe.github.io/helm-charts-hardened/ \ - --set global.spire.trustDomain="$TRUST_DOMAIN" \ - --set spire-server.federation.enabled="true" \ - --set spire-server.controllerManager.watchClassless="true" \ - --namespace spire \ - --create-namespace \ - --install \ - --wait \ - --wait-for-jobs \ - --timeout "15m" -``` - -### SPIRE Agent - -The SPIRE Agent serves as the local identity provider for workloads and has the following characteristics: - -- **Deployment Methods**: SPIRE Agents can be deployed in multiple ways depending on the infrastructure - -as DaemonSets in Kubernetes environments or as standalone services on VMs and bare metal servers. -SPIRE Helm chart deploys a K8s SPIRE Agent across all nodes by default. -- **Server Communication**: Agents establish connections to the SPIRE Server to obtain workload identities, -acting as intermediaries between workloads and the central identity authority. -- **Workload Services**: Agents perform workload attestation (verification of workload identity) and -distribute SVIDs (SPIFFE Verifiable Identity Documents) through the Workload API, enabling secure -identity management at the workload level. - -### Directory Services - -Directory components can be deployed in the trust domain and configured -to use SPIRE with or without federation: - -```yaml -# Example Directory Server configuration to use SPIRE. -# Add client to server trust domain. -dir: - apiserver: - spire: - enabled: true - trustDomain: ${SERVER_TRUST_DOMAIN} - federation: - - trustDomain: ${CLIENT_TRUST_DOMAIN} - bundleEndpointURL: https://${CLIENT_BUNDLE_ADDRESS} - bundleEndpointProfile: - type: https_spiffe - endpointSPIFFEID: spiffe://${CLIENT_TRUST_DOMAIN}/spire/server - trustDomainBundle: | - ${CLIENT_BUNDLE_CONTENT} - -# Example Directory Client configuration to use SPIRE. -dirctl: - spire: - enabled: true - trustDomain: ${CLIENT_TRUST_DOMAIN} -``` - -## Test Example - -This test setup relies on using [Kubernetes Kind](https://kind.sigs.k8s.io/) clusters -running in [Docker](https://www.docker.com/) to simulate Security Trust Model in -federated setups. In the following example, we will: - -- Setup Two Kubernetes Kind clusters (one for each trust domain) -- Deploy SPIRE Servers and Agents in each cluster -- Configure Federation to establish trust between the clusters -- Deploy Directory services to communicate securely using SPIFFE identities - -```mermaid -flowchart TD - subgraph DIR_Trust_Domain[**Trust Domain**: dir.example] - DIR_SPIRE_SERVER[SPIRE Server] - DIR_API_SERVER[DIR API Server] - DIRCTL_API_CLIENT[DIRCTL Admin Client] - DIR_SPIRE_AGENT1[SPIRE Agent K8s] - DIR_SPIRE_SERVER <--> DIR_SPIRE_AGENT1 - DIR_SPIRE_AGENT1 -->|"Workload API"| DIR_API_SERVER - DIR_SPIRE_AGENT1 -->|"Workload API"| DIRCTL_API_CLIENT - DIRCTL_API_CLIENT -->|"API Call"| DIR_API_SERVER - end - subgraph DIRCTL_Trust_Domain[**Trust Domain**: dirctl.example] - DIRCTL_SPIRE_SERVER[SPIRE Server] - DIRCTL_CLIENT[DIRCTL Client] - DIRCTL_SPIRE_AGENT1[SPIRE Agent K8s] - DIRCTL_SPIRE_SERVER <--> DIRCTL_SPIRE_AGENT1 - DIRCTL_SPIRE_AGENT1 -->|"Workload API"| DIRCTL_CLIENT - end - DIR_SPIRE_SERVER <-.->|"Federation (SPIFFE Bundle)"| DIRCTL_SPIRE_SERVER - DIRCTL_CLIENT -->|"API Calls"| DIR_API_SERVER - - style DIRCTL_CLIENT fill:#84c294,stroke:#333,stroke-width:2px - style DIR_API_SERVER fill:#84c294,stroke:#333,stroke-width:2px - style DIRCTL_API_CLIENT fill:#84c294,stroke:#333,stroke-width:2px -``` - -**Deployment Tasks** - -The test example uses [kubernetes-sigs/cloud-provider-kind](https://github.com/kubernetes-sigs/cloud-provider-kind) -to expose Kubernetes Directory and SPIRE as *LoadBalancer* services between clusters. - -```bash -## Fetch Directory source -git clone https://github.com/agntcy/dir -cd dir - -## Build all components -task build - -## Deploy full federation setup -task test:spire - -## Cleanup test environment -task test:spire:cleanup -``` - -For more details, see: - -- [Running a Federated Directory Instance](partner-prod-federation.md) - Connect to the public Directory network -- [Federation Profiles](federation-profiles.md) - https_web vs https_spiffe configuration -- [SPIRE Documentation](https://spiffe.io/docs/latest/spiffe-about/overview/) -- [SPIRE Federation Guide](https://spiffe.io/docs/latest/spire-helm-charts-hardened-advanced/federation/) diff --git a/docs/dir/validation.md b/docs/dir/validation.md deleted file mode 100644 index c35c47c..0000000 --- a/docs/dir/validation.md +++ /dev/null @@ -1,198 +0,0 @@ -# Validation - -The Directory enforces validation on all records before accepting them. -Validation is performed using the [OASF SDK](../oasf/oasf-sdk.md), which requires an OASF schema server URL for validation. - -### Configuration - -The Directory server validates records using an OASF schema URL. By default, it uses `https://schema.oasf.outshift.com`, but you can configure a different OASF instance: - -**Using environment variables:** - -```bash -# Use default OASF instance (https://schema.oasf.outshift.com) -task server:start - -# Use custom OASF instance -DIRECTORY_SERVER_OASF_API_VALIDATION_SCHEMA_URL=https://your-custom-oasf.com task server:start -``` - -**Using YAML configuration:** - -```yaml -# server.config.yml -oasf_api_validation: - schema_url: "https://schema.oasf.outshift.com" -listen_address: "0.0.0.0:8888" -``` - -!!! note - - The server itself does not have a built-in default schema URL. Deployment tools like Helm and Taskfile set `https://schema.oasf.outshift.com` as the default. When using Docker Compose or running the server binary directly, you must explicitly set the `DIRECTORY_SERVER_OASF_API_VALIDATION_SCHEMA_URL` environment variable. - -### Validation Behavior - -The Directory server uses API-based validation against the configured OASF schema server: - -- **Validation Method**: HTTP requests to the OASF schema server API. -- **Errors vs Warnings**: Only errors cause validation to fail. Warnings are returned but do not affect the validation result. -- **Schema Version Detection**: The schema version is automatically detected from each record's `schema_version` field. -- **Supported Versions**: The OASF SDK decoder supports specific schema versions (currently: 0.7.0, 0.8.0, and 1.0.0). Records using unsupported versions are not validated. -- **Unknown Classes**: Classes (such as modules, skills, and domains) not defined in the OASF schema are rejected with an error. Records must only use classes that are defined in the configured OASF schema instance. - -### OASF Instance Configurations - -While there is only one validation method (API validation), you can configure the Directory server to use different OASF instances. -The choice of OASF instance affects which records are accepted and how compatible your directory instance is with other directory instances. - -The following table shows which OASF instance configurations can exchange records with each other: - -| Instance Type | Can Pull From | Can Be Pulled By | -|--------------|---------------|------------------| -| **Official OASF Instance** | Official OASF instance only | Official OASF instance and custom instances with additional taxonomy | -| **Custom OASF Instance (Additional Taxonomy)** | Official OASF instance, custom instances with same extended taxonomy | Custom instances with same extended taxonomy only | -| **Custom OASF Instance (Changed Taxonomy)** | Custom instances with same changed taxonomy only | Custom instances with same changed taxonomy only | - -#### Official OASF Instance - -Using the official OASF instance (`https://schema.oasf.outshift.com`) provides the baseline for the official network. -Records validated here form the most strict, compatible set. - -**Configuration:** - -```yaml -oasf_api_validation: - schema_url: "https://schema.oasf.outshift.com" -``` - -#### Custom OASF Instance (Additional Taxonomy) - -Using a custom OASF instance with extensions that add to the taxonomy (with modules, skills, and domains that extend the official taxonomy but doesn't change or remove). - -Records using the extended taxonomy can only be pulled by nodes using the exact same extended taxonomy. - -**Configuration:** - -```yaml -oasf_api_validation: - schema_url: "https://your-custom-oasf-instance.com" -``` - -#### Custom OASF Instance (Changed Taxonomy) - -Using a custom OASF instance with extensions that modify the taxonomy (with modules, skills, and domains that change or remove from the official taxonomy). - -This approach is completely incompatible with all other options, can only work with nodes using the exact same changed taxonomy. - -**Configuration:** - -```yaml -oasf_api_validation: - schema_url: "https://your-custom-oasf-instance.com" -``` - -### Deploying a Local OASF Instance - -You can deploy a local OASF instance alongside the Directory server for testing or development purposes. - -#### Testing with Local OASF Server - -To test with a local OASF instance deployed alongside the directory server: - -1. Enable OASF in Helm values - - Edit `install/charts/dir/values.yaml` with the following configuration: - - ```yaml - apiserver: - oasf: - enabled: true - ``` - -2. Set schema URL to use the deployed OASF instance - - In the same file, set the schema URL to use the deployed OASF instance: - - ```yaml - apiserver: - config: - oasf_api_validation: - schema_url: "http://dir-ingress-controller.dir-server.svc.cluster.local" - ``` - - Replace `dir` with your Helm release name and `dir-server` with your namespace if different. - -3. Deploy the local OASF instance - - ```bash - task build - task deploy:local - ``` - -The OASF instance is deployed as a subchart in the same namespace and automatically configured for multi-version routing via ingress. - -#### Using a Locally Built OASF Image - -If you want to deploy with a locally built OASF image (e.g., containing `0.9.0-dev` schema files), you need to load the image into Kind before deploying. -The `task deploy:local` command automatically creates a cluster and loads images, but it doesn't load custom OASF images. - -Follow the steps below: - -1. Create the Kind cluster - - ```bash - task deploy:kubernetes:setup-cluster - ``` - - This creates the cluster and loads the Directory server images. - -2. Build your local OASF image with the `latest` tag - - ```bash - cd /path/to/oasf/server - task build - ``` - -3. Load the OASF image into Kind - - ```bash - kind load docker-image ghcr.io/agntcy/oasf-server:latest --name agntcy-cluster - ``` - -4. Configure `values.yaml` to use the local image: - - ```yaml - oasf: - enabled: true - image: - repository: ghcr.io/agntcy/oasf-server - versions: - - server: latest - schema: 0.9.0-dev - default: true - ``` - -5. Deploy the Directory - - Don't use `task deploy:local` as it will recreate the cluster. - - ```bash - task deploy:kubernetes:dir - ``` - -!!! note - - If you update the local OASF image, reload it into Kind and restart the deployment: - - ```bash - kind load docker-image ghcr.io/agntcy/oasf-server:latest --name agntcy-cluster - kubectl rollout restart deployment/dir-oasf-0-9-0-dev -n dir-server - ``` - -### Related Documentation - -For more information, see the following: - -- [OASF Validation Service](../oasf/validation.md) - Detailed validation service documentation -- [Validation Comparison](../oasf/validation-comparison.md) - Comparison between API validator and JSON Schema -- [OASF Extensions](https://github.com/agntcy/oasf/blob/main/CONTRIBUTING.md#oasf-extensions) - Information about creating OASF extensions diff --git a/docs/identity/.index b/docs/identity/.index deleted file mode 100644 index e0ce205..0000000 --- a/docs/identity/.index +++ /dev/null @@ -1,24 +0,0 @@ -nav: - - Introduction: identity.md - - Identifiers: identifiers.md - - Identity Service: - - Overview: identity_service.md - - Connecting an Identity Provider: connecting_idp.md - - Creating Agentic Service Identities: creating_identities.md - - Verifying an Identity: verify_identity.md - - Creating Policies for Agentic Services: creating_policies.md - - API Access: identity_service_api_access.md - - Identity Service API Reference: identity_service_api.md - - Protofiles: identity_service_protofiles.md - - Settings: identity_service_settings.md - - Identity Service SDK: identity_service_sdk.md - - Development Guide: identity_service_development.md - - Contributing Guide: identity_service_contributing.md - - Identifier Examples: identifier_examples.md - - Verifiable Credentials: credentials.md - - Agent Badge Examples: vc_agent_badge.md - - MCP Server Badge Examples: vc_mcp.md - - Architecture Diagrams: arch_diagrams.md - - Sequence Flows: flow.md - - OpenAPI Reference: openapi.md - - Identity Quickstart Guide: identity-quickstart.md \ No newline at end of file diff --git a/docs/identity/arch_diagrams.md b/docs/identity/arch_diagrams.md deleted file mode 100644 index 332d7b7..0000000 --- a/docs/identity/arch_diagrams.md +++ /dev/null @@ -1,54 +0,0 @@ -# Architecture Diagrams - -## Generating Agent Badges - -![Issuing Architecture Diagram](../assets/identity/issuing.png) - -The above diagram depicts the process for generating and storing a verifiable Agent Badge along with a `ResolverMetadata` object associated to an agent subject. A similar process can be followed to generate a verifiable MCP Server Badge or a verifiable MAS Badge. - -- The top left of the image shows that an organization may have several sub-organizations, each of which may create its own Agent subjects. -- As described in the [examples](./identifier_examples.md), each agent subject will be associated to an agent ID. More specifically, AGNTCY enables organizations to bring their own identities for their agents (for example, identities created via Okta, Duo, or A2A) as well as the capacity to generate an identity through AGNTCY, for example, using DIDs. -- Each agent subject is required to use a schema to create an Agent definition (e.g., using an [OASF schema](../oasf/open-agentic-schema-framework.md) or an A2A [Agent Card](https://a2a-protocol.org/latest/topics/key-concepts/)). The agent ID must be included in the Agent Definition (see examples of agent IDs [here](./identifier_examples.md)). -- Each organization or sub-organization may create public and private key pairs, and store the private keys (PrivKeys) on their wallet or vault of choice, or other means enabling secure access to the PrivKeys. -- A PrivKey owned and managed by an organization or sub-organization is required to create a ProofValue of the agent definition. As described in detail in the [Agent Badge Examples](./identifier_examples.md), the Agent Definition, the ProofValue, and additional metadata are used to create a Verifiable Credential in the form of a verifiable Agent Badge. Also note that, specific metadata, such as a verification method, an assertion method, and a service endpoint are used to create a `ResolverMetadata` object that is associated to the ID and the verifiable Agent Badge. - - !!! note - Embedding the PubKey in the `ResolverMetadata` object itself is optional, since the metadata supplied allows for automated access and verification of the PubKey (e.g., using a JWK as part of a JWT header). - -- The `ResolverMetadata` along with the verifiable Agent Badge can be stored in Identity Nodes (INs) that can operate as trust anchors. In subsequent updates to this documentation, AGNTCY shall provide more detailed recommendations about the INs, and their role and capacity to operate as decentralized trust anchors, especially, to: - - - Build trust during MAS composition involving third-party Agents and MCP Servers. - - Link and automate the dynamic and trustworthy discovery of running Agents and MCP Servers to their corresponding AuthN and delegated AuthZ methods, including MFA in a MAS. - -- AGNTCY considers the possibility that organizations and their sub-organizations may register with the INs (e.g., to brand and ensure the origin of their agents). This may include means to store and bind an organization/sub-organization to a PubKey, the IDs for the various subjects that they might register (e.g., Agents and MCP Servers), their corresponding `ResolverMetadata` objects and Agent Badges, as well as additional sets of VCs for each of them. - -AGNTCY plans to contribute open-source code to automate the process of creating and storing `ResolverMetadata` objects, Agent Badges, and MCP Server Badges. - -## Verifying Agent Badges - -![Verification Architecture Diagram](../assets/identity/verification.png) - -The above diagram depicts an example process enabling the lookup, identification and use of `ResolverMetadata` as well as verifying an Agent Badge associated to an Agent subject. The example process uses AGNTCY's [Agent Discovery Service](../dir/overview.md) as the means to discover an Agent with specific skills, and automatically identify and resolve the associated Agent Badge, irrespective of the type of identity used by the owner of the Agent (e.g., an Okta, Duo, AD, DID, or A2A ID). - -A similar process can be followed in the case of MCP Servers or A2A Agents, but in those cases the discovery service may rely on the use of well-known addresses (URLs), or other hubs, external directories, or discovery services. - -The following steps summarize the process: - -1. An Agent Consumer may search for an Agent with given skills, e.g., using an ADS client, such as a CLI, a UI, or a headless interface in case the consumer is an agent itself. -2. A lookup is performed using the ADS. -3. A list of potential candidates are presented to the Agent Consumer. -4. The Agent Consumer selects the desired Agent. -5. This now triggers the lookup of the selected Agent's definition (e.g., a definition based on an OASF schema), which contains the Agent ID (e.g., an Okta, Duo, AD, DID, or A2A ID). -6. The Agent definition is obtained and passed to the Resolver, which is one of the key elements of the AGNTCY's identity service. -7. The Resolver extracts the Agent ID from the Agent definition in step (7a), and performs a lookup process in an AGNTCY's Identity Node (IN) using the Agent ID as a key (see step (7b) in the diagram above). -8. The corresponding Agent Badge and ResolverMetadata object are now passed to the Resolver. -9. The Resolver provides trustworthy and automated means to resolve and verify the Agent subject. To this end, the Resolver proceeds as follows: - - - It uses the ResolverMetadata object to obtain the crypto method and PubKey to decrypt the ProofValue in the Agent Badge. - - It decrypts the ProofValue and verifies the Agent Badge integrity, since, as shown in the examples [here](./vc_agent_badge.md), the proof type is a `"DataIntegrityProof"`. This may include the computation of a digest, and comparison with a digest obtained after decryption. - - It outputs the verification result and logs it. - -10. The Verified Agent Badge is returned to the ADS client. -11. This is forwarded to the Agent Consumer. - -AGNTCY plans to contribute open-source code to automate the process of resolving and verifying Agent Badges and MCP Server Badges, leveraging `ResolverMetadata` objects. diff --git a/docs/identity/connecting_idp.md b/docs/identity/connecting_idp.md deleted file mode 100644 index 7d76ca0..0000000 --- a/docs/identity/connecting_idp.md +++ /dev/null @@ -1,175 +0,0 @@ -# Connecting an Identity Provider - -This document provides a comprehensive guide on how to register a new issuer by connecting an Identity Provider (IdP) within the **AGNTCY Identity Service**. -Connecting an IdP is a crucial step for integrating external authentication and authorization services, such as Client Credentials, with your AGNTCY Identity Service environment. -This guide specifically details the process for configuring different Identity Providers, including Duo and Ory. - -To access the Identity Provider creation page within the **AGNTCY Identity Service**: - -1. From the main dashboard, locate and click on **Settings** in the left-hand navigation menu. -2. Within the Settings section, select **Identity Provider**. -3. On the Identity Provider management page, click the **Connect** button to initiate the creation wizard. - -This will direct you to the "Identity Provider Connection" page, where you can begin configuring your new IdP. - -![Register Issuer](../assets/identity/identity_service/register-issuer.png) - -Now proceed to the next section for detailed instructions for each supported Identity Provider. - -## Connecting Duo as an Identity Provider - -This guide specifically details the process for configuring Duo as an Identity Provider. - -### Prerequisites - -Before you begin the Identity Provider connection process, ensure you have the following: - -- **Access to AGNTCY Identity Service:** You must have an administrator role or sufficient permissions within the AGNTCY Identity Service application to access the Settings and connect Identity Providers. -- **Duo Security Account:** An active Duo Security account is required. -- **Duo Application Details:** You must have an existing Duo application configured within your Duo Admin Panel. From this application, you will need to retrieve: - - Your Duo **API Hostname** - - The **Integration Key** for your Duo application - - The **Secret Key** for your Duo application - -!!! note - You can follow the [Duo Admin API documentation](https://duo.com/docs/adminapi) for detailed instructions on how to create and manage applications within Duo Security. - - Below you can find a screenshot of the Duo Admin API interface, with all the necessary permissions and the necessary fields for your integration. - -![Duo Admin API](../assets/identity/identity_service/duo-admin-view.png) - -### Identity Provider Connection Steps - -Follow these steps to configure and register your Identity Provider: - -1. **Select Identity Provider:** - - - On the "Identity Provider Connection" page, you will be presented with a selection of supported Identity Providers. - - Carefully choose the provider you intend to integrate: - - **Duo** (as shown in the example, for Duo Security integration) - - **Critical Note:** The selection of an Identity Provider is a **one-time** action. Once saved, this choice cannot be modified later. Ensure you select the correct provider before proceeding. - -2. **Enter Provider Details:** - - - After selecting your desired Identity Provider (e.g., Duo), the "Provider details" section will become active, prompting you for specific configuration parameters. - - **Hostname:** Enter the API hostname for your Duo Security account. This is your unique Duo API endpoint. - - _Example:_ `api-ffb0a31a.duosecurity.com` - - **Integration Key:** Input the Integration Key. This key uniquely identifies your application within Duo Security and is obtained from your Duo Admin Panel when you create or configure an application. - - _Example:_ `DI12VLEZPQIF5JH7F8G8` - - **Secret Key:** Provide the Secret Key. This is a sensitive credential used for cryptographic signing of requests to the Duo API, ensuring the authenticity and integrity of communications. It is also obtained from your Duo Admin Panel. - For security purposes, the input in this field will be masked (displayed as asterisks). - - ![Register Issuer With Duo](../assets/identity/identity_service/register-issuer-duo.png) - -3. **Save Configuration:** - - Once all required details (Hostname, Integration Key, and Secret Key) have been accurately entered, click the **Save** button. - - Upon successful saving, your chosen Identity Provider will be registered and configured within AGNTCY Identity Service. - - If you need to discard the entered information and cancel the creation process, click the **Cancel** button. - -![Register Issuer With Duo Success](../assets/identity/identity_service/register-issuer-duo-done.png) - -## Connecting Okta as an Identity Provider - -This guide specifically details the process for configuring Okta as an Identity Provider. - -### Prerequisites - -Before you begin the Identity Provider connection process, ensure you have the following: - -- **Access to AGNTCY Identity Service:** You must have an administrator role or sufficient permissions within the AGNTCY Identity Service application to access the Settings and connect Identity Providers. -- **Okta Platform Account:** An active Okta Platform account is required. -- **Okta Admin Management API access:** You must setup an Okta application configured within your Okta Admin Panel. From this application, you will need to retrieve: - - Your Okta **Organization URL** - - The **Client ID** for your Okta application - - The **Private Key in PEM format (base64 value)** for your Okta application - -![Okta Private Key Setup](../assets/identity/identity_service/okta_private_key_auth.png) -![Okta Private Key PEM](../assets/identity/identity_service/okta_private_key_pem.png) - -!!! note - You can follow the [OAuth guide for Okta](https://developer.okta.com/docs/guides/implement-oauth-for-okta/main/) for detailed instructions on how to create and setup the application within Okta Platform. - - Below you can find a screenshot of the Okta Admin Panel interface, with all the necessary permissions and the necessary fields for your integration. - -![Okta Admin Scopes](../assets/identity/identity_service/okta_admin_scopes.png) -![Okta Admin Role](../assets/identity/identity_service/okta_admin_role.png) - -### Identity Provider Connection Steps - -Follow these steps to configure and register your Identity Provider: - -1. **Select Identity Provider:** - - - On the "Identity Provider Connection" page, you will be presented with a selection of supported Identity Providers. - - Carefully choose the provider you intend to integrate: - - **Okta** (as shown in the example, for Okta integration) - - **Critical Note:** The selection of an Identity Provider is a **one-time** action. Once saved, this choice cannot be modified later. Ensure you select the correct provider before proceeding. - -2. **Enter Provider Details:** - - - After selecting your desired Identity Provider (e.g., Okta), the "Provider details" section will become active, prompting you for specific configuration parameters. - - **Organization URL:** Enter the Organization URL for your Okta account. This is your unique Okta API endpoint. - - _Example:_ `https://trial-2273708.okta.com/` - - **Client ID:** Input the Client ID. This key uniquely identifies your application within Okta and is obtained from your Okta Admin Panel when you create or configure an application. - - _Example:_ `0oawfccje6jmJdybZ697` - - **Private Key:** Provide the Private Key in PEM format (base64 value). This is a sensitive credential used for cryptographic signing of requests to the Okta API, ensuring the authenticity and integrity of communications. It is also obtained from your Okta Admin Panel. - For security purposes, the input in this field will be masked (displayed as asterisks). - _Example:_ `MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQD...` - - ![Register Issuer With Okta](../assets/identity/identity_service/register-issuer-okta.png) - -3. **Save Configuration:** - - Once all required details (Organization URL, Client ID, Private Key) have been accurately entered, click the **Save** button. - - Upon successful saving, your chosen Identity Provider will be registered and configured within AGNTCY Identity Service. - - If you need to discard the entered information and cancel the creation process, click the **Cancel** button. - -![Register Issuer With Okta Success](../assets/identity/identity_service/register-issuer-okta-done.png) - -## Connecting Ory as an Identity Provider - -This guide specifically details the process for configuring [Ory](https://console.ory.sh/) as an Identity Provider. - -### Prerequisites - -Before you begin the Identity Provider connection process, ensure you have the following: - -- **Access to AGNTCY Identity Service:** You must have an administrator role or sufficient permissions within the AGNTCY Identity Service application to access the Settings and connect Identity Providers. -- **Ory Account:** An active [Ory](https://console.ory.sh/) account is required. -- **Ory API Project Slug:** Navigate to your Ory Console and select project settings to find your project slug. -- **Ory API Key:** Create a new API key in your Ory Console. This key will be used to authenticate requests from AGNTCY Identity Service to Ory. - -!!! note - Ory accounts are free to create, and you can use them to manage your identity providers. - - You can follow the [Ory documentation](https://www.ory.sh/docs/category/operations-reference) for detailed instructions on how to create and manage projects and API keys within Ory. - - Below you can find screenshots of the Ory Console interface, showing where to find your project slug and how to create an API key. - -![Ory Project Slug](../assets/identity/identity_service/ory-project-settings-view.png) -![Ory API Key](../assets/identity/identity_service/ory-api-key-creation.png) - -### Identity Provider Connection Steps - -Follow these steps to configure and register your Identity Provider: - -1. **Select Identity Provider:** - - - On the "Identity Provider Connection" page, you will be presented with a selection of supported Identity Providers. - - Carefully choose the provider you intend to integrate: - - **Ory** (as shown in the example, for Ory integration) - - **Critical Note:** The selection of an Identity Provider is a **one-time** action. Once saved, this choice cannot be modified later. Ensure you select the correct provider before proceeding. - -2. **Enter Provider Details:** - - - After selecting your desired Identity Provider (e.g., Ory), the "Provider details" section will become active, prompting you for specific configuration parameters. - - **Project Slug:** Enter the Ory project slug. This is a unique identifier for your Ory project and can be found in your Ory Console under project settings. - - _Example:_ `mystifying-kapitsa-y0k3j7igbj` - - **API Key:** Provide the API Key. This key is used to authenticate requests from AGNTCY Identity Service to Ory and is generated in your Ory Console. - - ![Register Issuer With Ory](../assets/identity/identity_service/register-issuer-ory.png) - -3. **Save Configuration:** - - Once all required details (Hostname, Integration Key, and Secret Key) have been accurately entered, click the **Save** button. - - Upon successful saving, your chosen Identity Provider will be registered and configured within AGNTCY Identity Service. - - If you need to discard the entered information and cancel the creation process, click the **Cancel** button. - -![Register Issuer With Ory Success](../assets/identity/identity_service/register-issuer-ory-done.png) diff --git a/docs/identity/creating_identities.md b/docs/identity/creating_identities.md deleted file mode 100644 index 64556bb..0000000 --- a/docs/identity/creating_identities.md +++ /dev/null @@ -1,103 +0,0 @@ -# Creating Agentic Service Identities - -This guide outlines the process of creating and registering Agentic Services within the **AGNTCY Identity Service**, which involves establishing a unique identity for your service and subsequently issuing a badge for its discovery. - -## 1. Create Agentic Service (Identity Creation) - -The first step involves defining your Agentic Service, which automatically creates an identity for it using the Identity Provider and assigns a unique API Key. This API Key is crucial for subsequent operations and authentication. - -**Steps:** - -1. Navigate to **Agentic Services** in the left-hand menu. -2. Click on **Create Agentic Service**. -3. **Select Agentic Service Type & Source:** - - Choose the appropriate type for your service: - - **OASF:** For Open Agent Schema defined services. - - **MCP Server:** For services running on an MCP (Model Context Protocol) Server. _(As shown in the screenshot, this is the selected option.)_ - - **A2A Protocol:** For Application-to-Application protocol services. - - ![Create Agentic Service MCP Server](../assets/identity/identity_service/agentic-service-mcp.png) - -4. **Details:** - - **Name:** Enter a descriptive name for your Agentic Service (e.g., "Currency Exchange MCP Server"). This name will help you identify the service within the platform. - - **Description:** Provide a brief explanation of your service's purpose (e.g., "A Currency Exchange MCP Server"). - - ![Create Agentic Service MCP Server](../assets/identity/identity_service/agentic-service-mcp-2.png) - -5. Click **Next** to proceed. Upon creation, an API Key will be generated for your Agentic Service. Ensure you securely store this API Key, as it will be required for issuing the badge. - - ![Create Agentic Service MCP Server](../assets/identity/identity_service/agentic-service-mcp-no-badge.png) - -## 2. Issuing the Badge - -Issuing a badge enables the discovery of your Agentic Service, whether it's an MCP Server tool or an A2A Agent's well-known Agent Card. The method for issuing the badge depends on whether your service is accessible from the internet. - -### A. Service Accessible from the Internet - -If your Agentic Service can be accessed directly from the public internet, you can provide its deployment URL for online discovery. - -**Steps:** - -1. During the "Register Agentic Service" step, you will be prompted to provide the **Deployment URL** for your service. -2. Enter the full URL where your service is hosted and accessible. -3. The service will then perform online discovery using this URL. - -![Issue the Badge using the UI](../assets/identity/identity_service/issue-badge-online.png) - -### B. Service Not Accessible from the Internet (Including Localhost and Development Deployments) - -For services that are not directly accessible from the public internet (e.g., services running on localhost, private networks, or development environments), you can use the Python SDK to issue the badge. - -**Steps:** - -1. **Install the Python SDK:** - - Ensure you have the Python SDK installed. Refer to [the SDK section](./identity_service_sdk.md) of the documentation for detailed installation instructions. -2. **Perform the Badge Creation Command:** - - Open your terminal or command prompt. - - Execute the following command: - ```bash - identity-cli badge create {URL} - ``` - _Replace `{URL}` with the local or internal URL of your Agentic Service._ - - You will then be prompted to provide the **Agentic Service API Key** that was generated during the "Create Agentic Service" step. Enter the API Key when requested. - -![Issue the Badge using the CLI](../assets/identity/identity_service/issue-badge-offline.png) - -This command will facilitate the offline registration and badge issuance for your Agentic Service. - -Once the badge is successfully issued, it will be associated with your Agentic Service, allowing it to be discovered and utilized. -You can view the details of your newly created Agentic Service, including its API Key and badge status, in the Agentic Services dashboard: - -![All Agentic Services](../assets/identity/identity_service/agentic-service-all.png) - -And the details of the Agentic Service you just created: - -![Create Agentic Service MCP Server](../assets/identity/identity_service/agentic-service-mcp-badge.png) - -## 3. Policies Associated to An Agentic Service - -Located at the top right corner of the details screen of an Agentic Service, two tabs called "Policies Assigned" and "Policies Used By" allow you to see which Policies are currently assigned to the Agentic Service and which policies from this Agentic Service may be used by other Agentic Services respectively. - -### Policies Assigned - -The Policies Assigned tab provides an overview of the policies that are currently assigned to a specific agent service, such as the emailMCPServer. This section displays essential information including: - -- **Policy Name:** The name of the policy, such as "Invoke Policy". -- **Rules:** The specific rule(s) associated with each policy. -- **Assigned To:** Indicates the agent or service to which the policy is assigned, such as emailMCPServer. -- **Updated At & Created At:** Timestamp details indicating when the policy was last updated or created. -- **Action & Needs Approval:** Shows the allowed actions and whether approval is required or not. - -![Policies Assigned to Agentic Service](../assets/identity/identity_service/agentic-service-mcp-badge-policies-assigned.png) - -### Policies Used By - -The Policies Used By tab provides an overview of the policies used by the agent service. This section displays essential information including: - -- **Policy Name:** The name of the policy, such as "Invoke Policy". -- **Rules:** The specific rule(s) associated with each policy. -- **Assigned To:** Indicates the agent or service to which the policy is assigned, such as emailMCPServer. -- **Updated At & Created At:** Timestamp details indicating when the policy was last updated or created. -- **Action & Needs Approval:** Shows the allowed actions and whether approval is required or not. - -![Policies Assigned to Agentic Service](../assets/identity/identity_service/agentic-service-mcp-badge-policies-usedby.png) diff --git a/docs/identity/creating_policies.md b/docs/identity/creating_policies.md deleted file mode 100644 index d84a2cf..0000000 --- a/docs/identity/creating_policies.md +++ /dev/null @@ -1,87 +0,0 @@ -# Creating Policies for Agentic Services - -The Policies section in the **AGNTCY Identity Service** allows administrators to define rules and permissions for Agentic Services. -This guide will walk you through the process of creating, editing, and managing policies, -ensuring your Agentic Services operate within the desired constraints and capabilities. - -## Introducing Task-Based Access Control - -The **AGNTCY Identity Service** leverages Task-Based Access Control (TBAC) to enhance the management and security of Agentic Services. TBAC allows administrators to define specific tasks that can be performed by each service and the permissions required to execute these tasks. By integrating TBAC into your policies, you can ensure that each Agentic Service adheres to organizational security standards and operational requirements. - -### Benefits of TBAC - -- **Enhanced Security:** TBAC provides a granular level of security by defining task-specific permissions, reducing the risk of unauthorized access. -- **Flexibility:** Easily adjust permissions as organizational needs change, ensuring your Agentic Services remain adaptable to evolving requirements. -- **Centralized Management:** Administer all task permissions from a single interface within the Policies section, simplifying oversight and updates. - -## 1. Accessing the Policies Section - -1. **Navigate to Policies:** - - - From the main dashboard, click on the "Policies" section in the left-hand navigation menu to view existing policies or create new ones. - - ![Navigate to Policies section](../assets/identity/identity_service/policies_01.png) - -## 2. Creating a New Policy - -1. **Add Policy:** - - - Click on the "Add Policy" button to initiate the creation process. This will open the policy creation wizard. - -2. **Policy Details:** - - - **Name:** Enter a descriptive name for your policy (e.g., "Email Policy"). - - **Assigned To:** Select the Agentic Service the policy will apply to (e.g., "SuperAgent"). - - **Description:** Provide a brief description of the policy's purpose and scope. - - ![Policy Details](../assets/identity/identity_service/policies_02.png) - -3. **Policy Rules:** - - - **Add Rule:** Define the specific actions and permissions associated with the policy. - - **Name:** Specify the rule name (e.g., "Search & Read Emails"). - - **Tasks:** Choose the tasks this rule will allow or restrict (e.g., "gmail_find_email"). - - **Action:** Select the action type, such as "Allow" or "Deny". - - **Needs Approval:** Optionally, specify if this action requires additional approval. - - ![Policy Rule Creation](../assets/identity/identity_service/policies_03.png) - ![Policy Rule Tasks Selection](../assets/identity/identity_service/policies_04.png) - ![Policy Rule Submission](../assets/identity/identity_service/policies_05.png) - -4. **Add Multiple Rules:** - - - You can add additional rules by clicking "Add Rule" and repeating the steps above for each new rule. This is useful for complex policies requiring multiple permissions. - - ![Multiple Rule Addition](../assets/identity/identity_service/policies_06.png) - -5. **Review and Save Policy:** - - - Review all policy details and rules to ensure accuracy. Once satisfied, click "Save" to finalize and save the policy. - - ![Review Policy](../assets/identity/identity_service/policies_07.png) - ![Review Policy Rule Details](../assets/identity/identity_service/policies_08.png) - ![Created Policy](../assets/identity/identity_service/policies_09.png) - -## 3. Managing Policies - -1. **Viewing Policies:** - - - The Policies dashboard displays all existing policies, their assigned Agentic Service, and the creation date. Click on a policy to expand and view detailed rules. - - ![View Policy Details](../assets/identity/identity_service/policies_10.png) - -2. **Editing Policies:** - - - To modify an existing policy, select "Edit" from the options menu. Make necessary changes to details or rules, and then save. - - ![Edit Policy Button](../assets/identity/identity_service/policies_11.png) - ![Edit Policy Details](../assets/identity/identity_service/policies_12.png) - -3. **Deleting Policies:** - - - If a policy is no longer needed, select "Delete" from the options menu to remove it from the system. Confirm the deletion when prompted. - -## Best Practices - -- Regularly review and update policies to align with changing organizational needs and security requirements. -- Utilize the "Needs Approval" feature for actions with higher security implications to maintain oversight. diff --git a/docs/identity/credentials.md b/docs/identity/credentials.md deleted file mode 100644 index a1aca8b..0000000 --- a/docs/identity/credentials.md +++ /dev/null @@ -1,50 +0,0 @@ -# Verifiable Credentials - -AGNTCY supports various types of Verifiable Credentials (VCs). A verifiable credential is a structured and cryptographically verifiable way to express claims made by an issuer. These claims can pertain to: - -- Agent definitions (for example, an [OASF definition](../oasf/open-agentic-schema-framework.md) or an [A2A Agent Card](https://a2a-protocol.org/latest/topics/key-concepts/)) -- Deployment configurations -- Authorization assertions used in processes such as Multi-Factor Authentication (MFA) - -## Key VCs - -The identity framework conceived by AGNTCY allows not only to cryptographically bind an agent ID to an issuer, a public key and a proof of provenance but also the binding of the same agent ID to different definitions of the core agent, including different schemas, versions, locators, etc., as well as additional VCs that may be used during Multi-Factor authentication and authorization (MFA) processes. The same approach applies to MCP Servers. - -Among some of the key VCs within AGNTCY are the following: - -### Agent Badge - -An Agent Badge is an enveloped VC captured in the form of a JSON-LD object that represents a specific definition of an agent subject in the IoA. - -The definition follows a given schema (for example, an OASF definition or an A2A Agent Card schema). An agent subject can have multiple Agent Badges, each representing a different definition of the same core agent or agent subject. For instance, different software versions and/or patched releases of an agent will have different Agent Badges. - -The same applies if the agent's code is available in different forms (for example, if it can be used and composed using different types of artifacts, such as a Docker container image or a Python package), or if the source code can be reached at different sites or routing locators (through GitHub or sites like Hugging Face), and so on. - -Examples of an Agent Badge can be found [here](./vc_agent_badge.md). - -### MCP Server Badge - -An MCP Server Badge is an enveloped VC, captured in the form of a JSON-LD object, that represents a specific definition of an MCP Server subject in the IoA. The definition should follow a given schema (that is, a json specification following a similar approach to A2A's card but for MCP Servers). - -Like in the case of an agent subject, an MCP Server can have multiple MCP Server Badges, each representing a different definition of the same core MCP Server subject. For instance, different software versions and/or patched releases of an MCP Server will have different MCP Server Badges. - -A example of an MCP Server Badge can be found [here](./vc_mcp.md). - -## Use Cases - -The combined use of Badges (VCs) and ResolverMetadata enables automated and trustworthy validation of: - -- Issuer public keys, via assertion methods. -- Authenticity and integrity of credentials (Agent or MCP Server Badges). -- Entity provenance and update lineage (especially critical for secure versioning). - -## Benefits of this Model - -- Prevents impersonation of Agents, and MCP Server resources and tools by ensuring provenance can be verified. -- Enables secure versioning and traceability, supporting safe upgrades and patching. -- Facilitates advanced authentication and authorization workflows, including those involving: - - Dynamic trust establishment - - Machine-to-machine negotiation (with or without human input) - - Pre-connection credential validation - -These capabilities apply equally to Agents and MCP Servers, supporting trust-aware composition and interaction across the IoA. diff --git a/docs/identity/flow.md b/docs/identity/flow.md deleted file mode 100644 index 357e61f..0000000 --- a/docs/identity/flow.md +++ /dev/null @@ -1,304 +0,0 @@ -# Sequence Flows - -## Identity Flows - -### Initial Identity Setup - -```mermaid -sequenceDiagram -autonumber - -Agent Creator->>Identity CLI: Connect to wallet -activate Identity CLI -Identity CLI->>Wallet: Connect -activate Wallet -Wallet-->>Identity CLI: Connected -deactivate Wallet -Identity CLI-->>Agent Creator: Connected -deactivate Identity CLI - -Agent Creator->>Identity CLI: Create and store public and private keys -activate Identity CLI -Identity CLI->>Identity CLI: Create public and private keys -Identity CLI->>Wallet: Store keys -activate Wallet -Wallet-->>Identity CLI: Stored -deactivate Wallet -Identity CLI-->>Agent Creator: Keys created and stored -deactivate Identity CLI - -Agent Creator->>Identity CLI: Connect to Identity Node -activate Identity CLI -Identity CLI->>Identity Node: Connect to Identity Node -activate Identity Node -Identity Node-->>Identity CLI: Connected -deactivate Identity Node -Identity CLI->>Agent Creator: Connected -deactivate Identity CLI - -Agent Creator->>Identity CLI: Request to publish public key as well known -activate Identity CLI -Identity CLI->>Wallet: Get public key -activate Wallet -Wallet-->>Identity CLI: Public key -deactivate Wallet -Identity CLI->>Identity Node: Request to publish public key as well known -activate Identity Node -Identity Node-->>Identity CLI: Respond with verification uri action
to complete publishing -Identity CLI-->>Agent Creator: Respond with verification uri action to complete publishing -deactivate Identity CLI -Agent Creator->>Identity Node: Complete verification uri action (e.g., via browser) -Identity Node-->>Identity Node: Publish public key as well known -Identity Node-->>Agent Creator: Published public key as well known -deactivate Identity Node -``` - -## User Flows - -### Create a New Agent - -This sequence diagram illustrates the process of creating, publishing, and registering an Agent's metadata and identity information within the Agntcy ecosystem. - -```mermaid -sequenceDiagram -autonumber - -Agent Creator->>e.g. Github: Publish agent source
code - -Agent Creator->>Identity CLI: Create and publish ResolverMetadata with an Agent ID - -Agent Creator->>Directory CLI: Create Agent OASF with Agent ID in identity extension - -Agent Creator->>Directory CLI: Publish OASF - -Agent Creator->>Identity CLI: Issue and Publish an Agent Badge (Verifiable Credential) with OASF -``` - -### Update an Agent - -This sequence diagram illustrates the process of updating an existing Agent along with its associated metadata and identity information within the Agntcy ecosystem. - -```mermaid -sequenceDiagram -autonumber - -Agent Creator->>e.g. Github: Update and publish agent source
code - -Agent Creator->>Directory CLI: Update Agent OASF keeping the same
Agent ID in identity extension - -Agent Creator->>Directory CLI: Publish OASF - -Agent Creator->>Identity CLI: Issue and Publish a new Agent Badge (Verifiable Credential) with OASF -``` - -### Verify an Agent Locally - -This sequence diagram illustrates the local verification process of an Agent's authenticity, including its associated identity credentials, within the Agntcy ecosystem. - -```mermaid -sequenceDiagram -autonumber - -Agent Consumer->>Directory CLI: Discover and download the agent OASF - -Agent Consumer->>Agent Consumer: Extract the Agent ID from
the OASF identity extension - -Agent Consumer->>Identity CLI: Resolve the Agent ID to get the Agent Badges - -Agent Consumer->>Agent Consumer: Find the Agent Badge
that matches the OASF - -Agent Consumer->>Identity CLI: Verify the Agent Badge -``` - -### Verify an Agent Using Search Endpoint - -This sequence diagram illustrates the process of verifying an Agent's authenticity using a search endpoint within the Agntcy ecosystem. This approach allows the Agent Verifier to locate and validate the correct Agent Badge by querying directly with both the Agent ID and OASF. - -```mermaid -sequenceDiagram -autonumber - -Agent Consumer->>Directory CLI: Discover and download the agent OASF - -Agent Consumer->>Agent Consumer: Extract the Agent ID from
the OASF identity extension - -Agent Consumer->>Identity CLI: Search for the Agent Badge
for the Agent ID + OASF - -Agent Consumer->>Identity CLI: Verify the Agent Badge -``` - -## Detailed Flows - -### Create a New Agent - -```mermaid -sequenceDiagram -autonumber - -Agent Creator->>e.g. Github: Publish agent source
code -activate e.g. Github -e.g. Github-->>Agent Creator: Published -deactivate e.g. Github - -Agent Creator->>Identity CLI: Create and publish ResolverMetadata with an Agent ID -activate Identity CLI -Identity CLI->>Wallet: Get Public Key -activate Wallet -Wallet-->>Identity CLI: Public Key -deactivate Wallet -Identity CLI->>Identity Node: Create and publish ResolverMetadata with an Agent ID -activate Identity Node -Identity Node->>Identity Node: Generate a globally unique ID -Identity Node->>Identity Node: Create ResolverMetadata with Agent ID -Identity Node-->>Identity CLI: Created and published -deactivate Identity Node -Identity CLI-->>Agent Creator: Created and published -deactivate Identity CLI - -Agent Creator->>Directory CLI: Create Agent OASF with Agent ID in identity extension -activate Directory CLI -Directory CLI-->>Agent Creator: OASF -deactivate Directory CLI - -Agent Creator->>Directory CLI: Publish OASF -activate Directory CLI -Directory CLI->>Directory: Publish OASF -activate Directory -Directory-->>Directory CLI: Published OASF with
Catalogue ID (Digest) -deactivate Directory -Directory CLI-->>Agent Creator: Published OASF with Catalogue ID (Digest) -deactivate Directory CLI - -Agent Creator->>Identity CLI: Issue and Publish an Agent Badge
(Verifiable Credential) with OASF -activate Identity CLI -Identity CLI->>Identity CLI: Issue an Agent Badge (Verifiable Credential) with OASF -Identity CLI->>Wallet: Get Private Key -activate Wallet -Wallet-->>Identity CLI: Private Key -deactivate Wallet -Identity CLI->>Identity CLI: Generate Data Integrity proof and add to Agent Badge -Identity CLI->>Identity Node: Publish the Agent Badge
(/v1alpha1/vc/publish) -activate Identity Node -Identity Node-->>Identity CLI: Published -deactivate Identity Node -Identity CLI-->>Agent Creator: Issued and Published -deactivate Identity CLI - -``` - -### Update an Agent - -```mermaid -sequenceDiagram -autonumber - -Agent Creator->>e.g. Github: Update and publish agent source
code -activate e.g. Github -e.g. Github-->>Agent Creator: Published -deactivate e.g. Github - -Agent Creator->>Directory CLI: Update Agent OASF keeping the same
Agent ID in identity extension -activate Directory CLI -Directory CLI-->>Agent Creator: OASF -deactivate Directory CLI - -Agent Creator->>Directory CLI: Publish OASF -activate Directory CLI -Directory CLI->>Directory: Publish OASF -activate Directory -Directory-->>Directory CLI: Published OASF with
Catalogue ID (Digest) -deactivate Directory -Directory CLI-->>Agent Creator: Published OASF with Catalogue ID (Digest) -deactivate Directory CLI - -Agent Creator->>Identity CLI: Issue and Publish a new Agent Badge (Verifiable Credential) with OASF -activate Identity CLI -Identity CLI->>Identity CLI: Issue a new Agent Badge (Verifiable Credential) with OASF -Identity CLI->>Wallet: Get Private Key -activate Wallet -Wallet-->>Identity CLI: Private Key -deactivate Wallet -Identity CLI->>Identity CLI: Generate Data Integrity proof -Identity CLI->>Identity Node: Publish the Agent Badge
(/v1alpha1/vc/publish) -activate Identity Node -Identity Node-->>Identity CLI: Published -deactivate Identity Node -Identity CLI-->>Agent Creator: Issued and Published -deactivate Identity CLI -``` - -### Verify an Agent Locally - -```mermaid -sequenceDiagram -autonumber - -Agent Consumer->>Directory CLI: Discover and download the agent OASF -activate Directory CLI -Directory CLI->>Directory: Discover and download the agent OASF -activate Directory -Directory-->>Directory CLI: Downloaded OASF -deactivate Directory -Directory CLI->>Agent Consumer: Downloaded OASF -deactivate Directory CLI - -Agent Consumer->>Agent Consumer: Extract the Agent ID from
the OASF identity extension - -Agent Consumer->>Identity CLI: Resolve the Agent ID to get the Agent Badges -activate Identity CLI -Identity CLI->>Identity Node: Resolve the Agent ID to get the Agent Badges -activate Identity Node -Identity Node-->>Identity CLI: Agent Badges -deactivate Identity Node -Identity CLI-->>Agent Consumer: Agent Badges -deactivate Identity CLI - -Agent Consumer->>Agent Consumer: Find the Agent Badge
that matches the OASF - -Agent Consumer->>Identity CLI: Verify the Agent Badge -activate Identity CLI -Identity CLI->>Identity Node: Resolve the Agent ID to get the ResolverMetadata -activate Identity Node -Identity Node-->>Identity CLI: ResolverMetadata -deactivate Identity Node -Identity CLI->>Identity CLI: Verify the Agent Badge Data Integrity proof
using the ResolverMetadata public key -Identity CLI-->>Agent Consumer: Verified -deactivate Identity CLI -``` - -### Verify an Agent Using Search Endpoint - -```mermaid -sequenceDiagram -autonumber - -Agent Consumer->>Directory CLI: Discover and download the agent OASF -activate Directory CLI -Directory CLI->>Directory: Discover and download the agent OASF -activate Directory -Directory-->>Directory CLI: Downloaded OASF -deactivate Directory -Directory CLI->>Agent Consumer: Downloaded OASF -deactivate Directory CLI - -Agent Consumer->>Agent Consumer: Extract the Agent ID from
the OASF identity extension - -Agent Consumer->>Identity CLI: Search for the Agent Badge
for the Agent ID + OASF -activate Identity CLI -Identity CLI->>Identity Node: Search for the Agent Badge
for the Agent ID + OASF
(/v1alpha1/vc/search) -activate Identity Node -Identity Node-->>Identity CLI: Agent Badge -deactivate Identity Node -Identity CLI-->>Agent Consumer: Agent Badge -deactivate Identity CLI - -Agent Consumer->>Identity CLI: Verify the Agent Badge -activate Identity CLI -Identity CLI->>Identity Node: Resolve the Agent ID to get the ResolverMetadata -activate Identity Node -Identity Node-->>Identity CLI: ResolverMetadata -deactivate Identity Node -Identity CLI->>Identity CLI: Verify the Agent Badge Data Integrity proof
using the ResolverMetadata public key -Identity CLI-->>Agent Consumer: Verified -deactivate Identity CLI -``` diff --git a/docs/identity/identifier_examples.md b/docs/identity/identifier_examples.md deleted file mode 100644 index 2572d08..0000000 --- a/docs/identity/identifier_examples.md +++ /dev/null @@ -1,220 +0,0 @@ -# Examples - -## IdP Examples (Okta and Duo) - -### Okta Example of an Agent ID - -#### ID - -```text -ID: OKTA-APP_ID -``` - -Where `ID` represents a universally unique identifier associated to an agent subject (e.g., an Okta Application ID in this case). - -#### ResolverMetadata - -The `ResolverMetadata` is represented as a JSON-LD object comprising the following elements: - -```json -ResolverMetadata -{ - id: "OKTA-APP_ID", - assertionMethod: [{ - publicKeyJwk: {} - }], - service: [{ - serviceEndpoint: "https://OKTA_TENANT_NAME.okta.com/" - }] -} -``` - -Where: - -- `assertionMethod`: contains the method, e.g., a JSON Web key (JWK), and in some cases, may also contain the public key that can be used to verify the [`Verifiable Credentials`](./credentials.md). JWKs are commonly used for signing and verifying JWTs (JSON Web Tokens). - - !!! note - While a JWK typically contains the crypto material encoding the public key itself (e.g., the RSA's modulus and exponent), in practice, JWKs are often retrieved dynamically from a JWKS (JSON Web Key Set) endpoint. More specifically, a JWKS is a collection of JWKs hosted by an authentication provider, allowing clients to fetch the appropriate key to verify JWTs without storing them manually. This is precisely the role of the `serviceEndpoint` below. - -- `serviceEndpoint`: The endpoint where JWKs can be dynamically retrieved from in case Okta is used. - -### Duo Example of an Agent ID - -#### ID - -```text -ID: DUO-CLIENT_ID -``` - -Where `ID` represents a universally unique identifier associated to an agent subject (e.g., a Duo client ID in this case). - -#### ResolverMetadata - -The `ResolverMetadata` is represented as a JSON-LD object comprising the following elements: - -```json -ResolverMetadata -{ - id: "DUO-CLIENT_ID", - assertionMethod: [{ - publicKeyJwk: {} - }], - service: [{ - serviceEndpoint: "https://api-(tenantid).duosecurity.com/" - }] -} -``` - -Where: - -- `assertionMethod`: Idem to the case above described for Okta. -- `serviceEndpoint`: The endpoint where JWKs can be dynamically retrieved from in case Duo is used. - -## A2A Example - -Although an Agent Card in A2A includes authentication requirements for an already deployed instance of an agent subject, such as basic or bearer authentication schemes, these are fundamentally focused on API authentication. More specifically, A2A has not yet addressed the problem of proving the provenance and integrity of an Agent Card. This is relevant given that an Agent Card comprises claims about the agent's version, its capabilities and skills, and other features, for which it is essential to build trust not only during the discovery phase (i.e., even before attempting to connect to the agent associated to the Agent Card) but also during the selection and composition of a Multi-Agent System (MAS). - -Hence, the A2A model may benefit from the use of verifiable identities and `ResolverMetadata` as detailed below. - -### ID - -```text -ID: A2A-Agent_ID -``` - -In the [Agent2Agent (A2A) model](https://a2a-protocol.org/latest/#/documentation), the `ID` could be represented by a URL, e.g., hosted at `https://YOUR-DOMAIN/.well-known/agent.json`, which links to a structured metadata file in the form of an [Agent Card](https://a2a-protocol.org/latest/topics/key-concepts/). - -### ResolverMetadata - -A `ResolverMetadata` example for an A2A agent represented as a JSON-LD object: - -```json -ResolverMetadata -{ - id: "A2A-Agent_ID", - assertionMethod: [{ - publicKeyJwk: {} - }], - service: [{ - serviceEndpoint: "https://api.NODE/ORG" - }] -} -``` - -Where: - -- `assertionMethod`: Idem to the cases above described for Okta and Duo. -- `serviceEndpoint`: The endpoint where JWKs can be dynamically retrieved from. In this case, this could be a **trust anchor**, e.g., an Identity Node within the AGNTCY identity system. - -## MCP Server Examples - -The latest MCP specification covers authentication and delegated authorization requirements and recommends the use of OAuth 2.1, which entails the use of IdPs and Auth Providers, such as Okta, Duo, or others. However, MCP has not yet addressed the problem of proving provenance and building trust during dynamic discovery and selection of already deployed MCP Servers. For instance, a calling agent might want to verify the provenance and resources and/or tools supported by an MCP Server, and build trust even before attempting to connect to it (i.e., building trust even before the connectivity and authentication process is started). - -Hence, the MCP model may benefit from the use of an [MCP Badge](./credentials.md) and `ResolverMetadata` in order to automatically discover public keys and verify their origin in a trusted manner. The examples below show IDs and `ResolverMetadata` for MCP Servers when Okta or Duo are used as IdPs. -Also note that, the AGNTCY enables organizations to bring their own MCP Server IDs (as in the examples 3.a) and 3.b) below) or create new ones via the AGNTCY identity services. Hence, the MCP Server identity might be a Fully Qualified Domain Name (FQDN), an ID created through Okta, Duo, AD, Entra ID or other identity providers, or a DID. - -### Okta Example of an MCP Server ID - -In order to enable the use of `ResolverMetadata` and generate MCP Server Badges that can be automatically resolved and verified when Okta is used as the IdP, the ID and `ResolverMetadata` associated to an MCP Server could follow the same approach as in the example 1.a) above. - -#### ID - -```text -ID: OKTA-APP_ID -``` - -Where `ID` represents a universally unique identifier associated to an MCP Server subject when Okta is used. - -#### ResolverMetadata - -The `ResolverMetadata` is represented as a JSON-LD object comprising the following elements: - -```json -ResolverMetadata -{ - id: "OKTA-APP_ID", - assertionMethod: [{ - publicKeyJwk: {} - }], - service: [{ - serviceEndpoint: "https://OKTA_TENANT_NAME.okta.com/" - }] -} -``` - -Where: - -- `assertionMethod`: Idem as in case 1.a). -- `serviceEndpoint`: The endpoint where the JWK associated to the MCP Server can be dynamically retrieved from in case Okta is used. - -### Duo Example of an MCP Server ID - -Likewise, to enable the use of `ResolverMetadata` and MCP Server Badges that can be automatically resolved and verified when Duo is used as the IdP, the ID and `ResolverMetadata` associated to an MCP Server could follow the same approach as in the example 1.b) above. - -#### ID - -```text -ID: DUO-CLIENT_ID -``` - -Where `ID` represents a universally unique identifier associated to an MCP Server subject when Duo is used. - -#### ResolverMetadata - -The `ResolverMetadata` is represented as a JSON-LD object comprising the following elements: - -```json -ResolverMetadata -{ - id: "DUO-CLIENT_ID", - assertionMethod: [{ - publicKeyJwk: {} - }], - service: [{ - serviceEndpoint: "https://api-(tenantid).duosecurity.com/" - }] -} -``` - -Where: - -- `assertionMethod`: Idem as in case 1.b). -- `serviceEndpoint`: The endpoint where the JWK associated to the MCP Server can be dynamically retrieved from in case Duo is used. - -## Decentralized Identifiers (DIDs) Example - -### ID - -```text -did:agntcy:{ID} -``` - -Where in this case, the `ID` is a `DID`. As indicated in the above `ID`, a `DID` structure is composed of three parts, providing a universally unique identifier that identifies the agent subject. - -### ResolverMetadata - -In this case, the `ResolverMetadata` is, according the [standard](https://www.w3.org/TR/did-1.1/), a DID Document. It is also represented as a JSON-LD object comprising the following elements: - -```json -ResolverMetadata -{ - id: "did:agntcy:ID", - verificationMethod: [{ - controller: "did:jwk:eyJhbGciOiJFUz....", - publicKeyJwk: {} - }], - assertionMethod: [{ - controller: "did:jwk:eyJhbGciOiJFUz....", - publicKeyJwk: {} - }], - service: [{ - serviceEndpoint: "https://api.NODE/ORG" - }] -} -``` - -Where: - -- `verificationMethod`: contains the public key that can be used to prove and verify the signatures, including ownership of a credential. -- `assertionMethod`: represents how a DID subject can issue or assert claims about themselves or others. For example, an entity might use assertion methods to sign verifiable credentials, proving that certain information (like an agent skill) is valid and trustworthy. -- `serviceEndpoint`: The endpoint or Identity Node where the DID Document is published and accessible from. diff --git a/docs/identity/identifiers.md b/docs/identity/identifiers.md deleted file mode 100644 index 07fe7f1..0000000 --- a/docs/identity/identifiers.md +++ /dev/null @@ -1,15 +0,0 @@ -# Identifiers - -## Definitions - -AGNTCY supports various types of identities, referred to as IDs, which serve as universally unique identifiers for the main entities or subjects operated by AGNTCY, including Agents, MCP Servers, and MASs. - -## Key Identifiers - -Each ID is associated 1:1 with `ResolverMetadata`, which contains the necessary information to establish trust while trying to use or interact with an Agent, an MCP Server, or a MAS ID. - -- ID: A universally unique identifier that represents the identity of a subject. - -- `ResolverMetadata`: Metadata, represented in the form of a JSON-LD object, containing cryptographic material and assertion methods enabling one to automatically resolve and establish trust with the associated ID (for example, an Agent, an MCP Server, or a MAS). - -Concrete examples with various `IDs` and associated `ResolverMetadata` can be found [`here`](./identifier_examples.md). diff --git a/docs/identity/identity-quickstart.md b/docs/identity/identity-quickstart.md deleted file mode 100644 index acbbb80..0000000 --- a/docs/identity/identity-quickstart.md +++ /dev/null @@ -1,243 +0,0 @@ -# Identity Quick Start Guide - -## Get Started - -This short guide allows you to setup the Identity `Issuer CLI` as well as the Identity `Node Backend`. -The `Issuer CLI` allows to generate, register, search for, and verify badges for Agents and MCP Servers. The CLI includes a library enabling storage and retrieval of the keys required to sign the badges, both on local storage or using a 3rd party wallet or vault. -The `Node Backend` comprises the APIs and the backend core. It stores, maintains, and binds org:sub-org IDs, PubKeys, Subject IDs and metadata, including badges, ResolverMetadata and Verifiable Credentials (VCs). - -For detailed information in the Agent Directory Service, see the [Identity documentation](../identity/identity.md). - -### Prerequisites - -To run these steps successfully, you need to have the following installed: - -- [Docker Desktop](https://docs.docker.com/get-docker/), or have both: [Docker Engine v27 or higher](https://docs.docker.com/engine/install/) and [Docker Compose v2.35 or higher](https://docs.docker.com/compose/install/) - -### Step 1: Install the Issuer CLI - -Use the following command to install the `Issuer CLI`: - -using `curl`: - -```bash -sh -c "$(curl -sSL https://raw.githubusercontent.com/agntcy/identity/refs/heads/main/deployments/scripts/identity/install_issuer.sh)" -``` - -or using `wget`: - -```bash -sh -c "$(wget -qO- https://raw.githubusercontent.com/agntcy/identity/refs/heads/main/deployments/scripts/identity/install_issuer.sh)" -``` - -!!! note - > You can also download the `Issuer CLI` binary corresponding to your platform from the [latest releases](https://github.com/agntcy/identity/releases). - > - > On some platforms you might need to add execution permissions and/or approve the binary in `System Security Settings`. - > - > For easier use, consider moving the binary to your `$PATH` or to the `/usr/local/bin` folder. - -If you have `Golang` set up locally, you could also use the `go install command`: - -```bash -go install github.com/agntcy/identity/cmd/issuer@latest && \ - ln -s $(go env GOPATH)/bin/issuer $(go env GOPATH)/bin/identity -``` - -### Step 2: Start the Node Backend with Docker - -1. Clone the repository: - - ```bash - git clone https://github.com/agntcy/identity.git - ``` - -2. Start the Node Backend with Docker: - - ```bash - ./deployments/scripts/identity/launch_node.sh - ``` - - Or use `make` if available locally: - - ```bash - make start_node - ``` - -!!! note - > You can also install the `Node Backend` using our helm chart, for which instructions are available in the [chart's](https://github.com/agntcy/identity/tree/main/charts/identity-node/README.md) directory. - -### Step 3: Verify the Installation - -You can verify the installation by running the command below to see the [different commands available](#core-commands-to-use-the-cli): - -```bash -identity -h -``` - -## Core commands to use the CLI - -Here are the core commands you can use with the CLI - -- **vault**: Manage cryptographic vaults and keys -- **issuer**: Register and manage issuer configurations -- **metadata**: Generate and manage metadata for identities -- **badge**: Issue and publish badges for identities -- **verify**: Verify identity badges -- **config**: Display the current configuration context - -## Run the demo - -This demo scenario will allow you to see how to use the AGNTCY Identity components can be used in a real environment. -You will be able to perform the following: - -- Register as an Issuer -- Generate metadata for an MCP Server -- Issue and publish a badge for the MCP Server -- Verify the published badge - -### Prerequisites - -First, follow the steps in the [Get Started](#get-started) section above to install the `Issuer CLI` and run the `Node Backend`, and generate a local vault and keys. - -To run this demo setup locally, you need to have the following installed: - -- [Docker Desktop](https://docs.docker.com/get-docker/), or have both: [Docker Engine v27 or higher](https://docs.docker.com/engine/install/) and [Docker Compose v2.35 or higher](https://docs.docker.com/compose/install/) -- [Ollama CLI](https://ollama.com/download) -- [Okta CLI](https://cli.okta.com/manual/#installation) - -### Step 1: Run the Samples with Ollama and Docker - -The agents in the samples rely on a local instance of the Llama 3.2 LLM to power the agent's capabilities. -With Ollama installed, you can download and run the model (which is approximately 2GB, so ensure you have enough disk space) using the following command: - -1. Run the Llama 3.2 model: - - ```bash - ollama run llama3.2 - ``` - -2. From the root of the repository, navigate to the `samples` directory and run the following command to deploy the `Currency Exchange A2A Agent` leveraging the `Currency Exchange MCP Server`: - - ```bash - cd samples && docker compose up -d - ``` - -3. [Optional] Test the samples using the provided [test clients](https://github.com/agntcy/identity/tree/main/samples/README.md#testing-the-samples). - -### Step 2: Use the CLI to create a local Vault and generate keys - -1. Create a local vault to store generated cryptographic keys: - - ```bash - identity vault connect file -f ~/.identity/vault.json -v "My Vault" - ``` - -2. Generate a new key pair and store it in the vault: - - ```bash - identity vault key generate - ``` - -### Step 3: Register as an Issuer - -For this demo we will use Okta as an IdP to create an application for the Issuer. -The quickly create a trial account and application, we have provided a script to automate the process using the Okta CLI. - -!!! important - > If you already have an Okta account, you can use the `okta login` command to log in to your existing organization. - > - > If registering a new Okta developer account fails, proceed with manual trial signup and then use the `okta login` command, - > as instructed by the Okta CLI. - -1. Run the following command from the root repository to create a new Okta application: - - ```bash - . ./demo/scripts/create_okta_app - ``` - -2. In the interactive prompt, choose the following options: - - `> 4: Service (Machine-to-Machine)`, `> 5: Other` - -3. Register the Issuer using the `Issuer CLI` and the environment variables from the previous step: - - ```bash - identity issuer register -o "My Organization" \ - -c "$OKTA_OAUTH2_CLIENT_ID" -s "$OKTA_OAUTH2_CLIENT_SECRET" -u "$OKTA_OAUTH2_ISSUER" - ``` - -!!! note - > You can now access the `Issuer's Well-Known Public Key` at [`http://localhost:4000/v1alpha1/issuer/{common_name}/.well-known/jwks.json`](http://localhost:4000/v1alpha1/issuer/{common_name}/.well-known/jwks.json), - > where `{common_name}` is the common name you provided during registration. - -### Step 4: Generate metadata for an MCP Server - -Create a second application for the MCP Server metadata using Okta, similar to the previous step: - -1. Run the following command from the root repository to create a new Okta application: - - ```bash - . ./demo/scripts/create_okta_app - ``` - -2. In the interactive prompt, choose the following options: - - `> 4: Service (Machine-to-Machine)`, `> 5: Other` - -3. Generate metadata for the MCP Server using the `Issuer CLI` and the environment variables from the previous step: - - ```bash - identity metadata generate -c "$OKTA_OAUTH2_CLIENT_ID" \ - -s "$OKTA_OAUTH2_CLIENT_SECRET" -u "$OKTA_OAUTH2_ISSUER" - ``` - -!!! note - > When successful, this command will print the metadata ID, which you will need in the next step to view published badges that are linked to this metadata. - -### Step 5: Issue and Publish a Badge for the MCP Server - -1. Issue a badge for the MCP Server: - - ```bash - identity badge issue mcp -u http://localhost:9090 -n "My MCP Server" - ``` - -2. Publish the badge: - - ```bash - identity badge publish - ``` - -!!! note - > You can now access the `VCs as a Well-Known` at [`http://localhost:4000/v1alpha1/vc/{metadata_id}/.well-known/vcs.json`](http://localhost:4000/v1alpha1/vc/{client_id}/.well-known/vcs.json), - > where `{metadata_id}` is the metadata ID you generated in the previous step. - -### (Optional) Step 6: Verify a Published Badge - -You can use the `Issuer CLI` to verify a published badge any published badge, not just those that you issued yourself. -This allows others to verify the Agent and MCP badges you publish. - -1. Download the badge that you created in the previous step, replacing {metadata_id} with the metadata ID from step 4: - - ```bash - curl -o vcs.json http://localhost:4000/v1alpha1/vc/{metadata_id}/.well-known/vcs.json - ``` - -2. Verify the badges using the `Issuer CLI`: - - ```bash - identity verify -f vcs.json - ``` -!!! note - > You can also use our Python SDK to verify the badge programmatically. See the [Python SDK](https://github.com/agntcy/identity/tree/main/sdk/python/README.md) for more details. - -## Development - -For more detailed development instructions please refer to the following sections: - -- [Node Backend](https://github.com/agntcy/identity/tree/main/cmd/node/README.md) -- [Issuer CLI](https://github.com/agntcy/identity/tree/main/cmd/issuer/README.md) -- [Samples](https://github.com/agntcy/identity/tree/main/samples/README.md) -- [Api Spec](https://github.com/agntcy/identity/tree/main/api/spec/README.md) -- [Node Client SDK](https://github.com/agntcy/identity/tree/main/api/client/README.md) diff --git a/docs/identity/identity.md b/docs/identity/identity.md deleted file mode 100644 index 3cd9737..0000000 --- a/docs/identity/identity.md +++ /dev/null @@ -1,83 +0,0 @@ -# Identity - -Secure and reliable communication between software agents is a cornerstone of the Internet of Agents (IoA) vision. Just as humans rely on official identification (for example, driver's licenses) for trusted real-world interactions, agents in a digital ecosystem require verifiable credentials to authenticate their identity. - -Without proper identity management, malicious or unverified agents can infiltrate Multi-Agent Systems (MASs), leading to misinformation, fraud, or security breaches. To mitigate these risks, AGNTCY provides a standardized and consistent framework for authenticating agents and validating associated metadata. - -This framework applies equally to: - -- Agents -- Model Context Protocol (MCP) Servers -- MAS - -Each plays a critical role in the architecture and operation of an IoA and must comply with shared identity and authentication principles. - -## Requirements - -AGNTCY defines identity assignment with the following core properties: - -- **Open:** No centralized authority is required for assigning identities. -- **Collision-free:** Each entity (Agent, MCP Server, or MAS) has a universally unique identifier. -- **Verifiable:** Each entity is backed by a Verifiable Credential (VC) that can be used to authenticate its ID and provenance. - -## Assignment Approaches - -- AGNTCY supports both conventions and standards for identity assignment. -- This approach promotes interoperability across varied systems and ecosystems. -- This also allows for flexible integration while ensuring consistent methods for assigning and verifying identifiers. - -### Conventions - -AGNTCY currently supports the following two types of conventions for identity assignment. - -#### Identity Provider Accounts - -The use of **User Accounts** or **Service Accounts** provided by an Identity Provider (IdP). In the context of AGNTCY, the following identity providers can be used to assign universally unique identifiers: - -- Okta -- Microsoft AD -- Entra ID -- Duo -- Ping Identity -- Auth0 -- Google ID - -#### Well-Known Identifiers - -The use of **well-known identifiers**, that is, following the convention proposed by Google's Agent2Agent (A2A) protocol. This convention enables both open identity assignment as well as the use of universally unique identifiers for Agent Cards, which capture the metadata and characteristics that define, allow to discover, and identify an Agent within the A2A ecosystem. - -More specifically, in the A2A protocol, the Agent Card standardizes the format of the data shared during discovery processes, which may be facilitated by hosting the Agent Card at a well-known path or identifier, such as: `https://YOUR-DOMAIN/.well-known/agent.json`. In this case, the Agent Card includes details such as the Agent's capabilities, authentication requirements, and endpoint information. - -### Standards - -AGNTCY supports the use of W3C [Decentralized Identifiers (DIDs)](https://www.w3.org/TR/did-1.1/) and associated standards, including: - -- DID Documents for managing identifier metadata. -- [Verifiable Credentials (VCs)](https://www.w3.org/TR/vc-data-model-2.0/) for cryptographic validation. -- Decentralized networks for publishing and resolving identities - -This approach enables: - -- Decentralized identity management. -- Multi-factor Authentication and Authorization processes across these entities as well as between them and humans. -- Verifiable attributes (skills, versions, roles, etc.). - -!!! note - Independently of whether the identity is assigned following a convention or a standardized framework, at this stage the main focus of the AGNTCY project is to provide a common and trustworthy mechanism to present identifiers and to verify them. More specifically, AGNTCY not only supports various (and heterogeneous) identifiers that are universally unique but also proposes a standard way of presenting and verifying such identifiers, thereby enabling freedom in the selection of interoperable identities in an IoA. - -## Agent Identity Structure - -![Agent Badge](../assets/identity/agent_badge.png) - -The figure above depicts the main elements of an Agent's subject identifier: - -- Each Agent subject has a universally unique identifier named [ID](./identifiers.md). -- Each ID is associated 1:1 to a [`ResolverMetadata`](./identifiers.md) object, enabling automated resolution and trustworthy verification of Agent IDs. -- Each `ID` is also associated 1:n to an [Agent Badge](./vc_agent_badge.md). - -For more information, see the [Examples](./identifier_examples.md). - -In AGNTCY, an Agent subject is tied to a unique identifier linked to one or more Verifiable Credentials (VCs), which contain information about the Agent, such as its ID, a schema definition (for example, an [OASF schema](../oasf/open-agentic-schema-framework.md)), and other metadata used for defining locators, authentication, MFA, and so on. Agents can use this Badge for secure presentation, verification, and enabling trust across multi-agent systems. - -!!! note - This same structure applies to MCP Servers and MASs, ensuring consistency across all identity-bearing entities in the IoA. diff --git a/docs/identity/identity_service.md b/docs/identity/identity_service.md deleted file mode 100644 index d440b48..0000000 --- a/docs/identity/identity_service.md +++ /dev/null @@ -1,119 +0,0 @@ -# Identity Service - -## What is the AGNTCY Identity Service? - -AGNTCY Identity Service serves as the central hub for managing and verifying digital identities for your Agentic Services. In today's interconnected digital landscape, secure and reliable identity management is paramount. AGNTCY Identity Service addresses this need by providing a streamlined service to: - -- **Verify** the authenticity of existing identity badges. -- **Register** new Agentic Services, establishing their unique identities. - -Whether you are integrating existing services or deploying new ones, AGNTCY Identity Service ensures that all your components—including MCP Servers, A2A Agents, and OASF—are properly identified and managed. - -## Getting Started with AGNTCY Identity Service - -To begin using AGNTCY Identity Service's features, you have two primary pathways: - -1. **Verify Existing Identities**: If you already possess identity badges for your Agentic Services, you can use AGNTCY Identity Service to verify their authenticity and integrate them into the system. -2. **Register New Agentic Services**: For new deployments or services that require a fresh identity, AGNTCY Identity Service facilitates the registration process, allowing you to create and manage their digital presence. - -### Get Started in 5 Minutes - -This short guide allows you to setup the Identity Service `Frontend` as well as the Identity Service `Backend`. - -#### Prerequisites - -To run these steps successfully, you need to have the following installed: - -- [Docker Desktop](https://docs.docker.com/get-docker/), or have both: [Docker Engine v27 or higher](https://docs.docker.com/engine/install/) and [Docker Compose v2.35 or higher](https://docs.docker.com/compose/install/) - -#### Setup OIDC Provider - -1. Setup OIDC Provider - - - Create an OIDC application in your OIDC provider. - - You can use any OIDC provider of your choice. For testing purposes, you can use [Ory](https://www.ory.sh/), [Keycloak](https://www.keycloak.org/) or [Auth0](https://auth0.com/). - Configure the following variables in your shell environment: - - ```bash - export OIDC_ISSUER_URL= - export OIDC_CLIENT_ID= - export OIDC_LOGIN_URL= - export OIDC_CLIENT_ID_CLAIM_NAME= - ``` - - where: - - - `OIDC_ISSUER_URL` - The URL of your OIDC provider (e.g., `https://{INSTANCE_URL}/oauth2/{CLIENT_ID}/.well-known/openid-configuration`). - - `OIDC_CLIENT_ID` - The client ID you created in your OIDC provider. - - `OIDC_LOGIN_URL` - The login URL of your OIDC provider (e.g., `https://{INSTANCE_URL}/oauth2/{CLIENT_ID}/authorize`). - - `OIDC_CLIENT_ID_CLAIM_NAME` - The claim name in the Access token that contains the client ID (default: `cid`). - - !!! note - Make sure to add `http://localhost:5500` as a redirect URI for your OIDC client. - - - Or use our demo script to setup a local OIDC provider using [Ory Hydra](https://www.ory.sh/): - - ```bash - . ./demo/scripts/setup_hydra_oidc - ``` - - This will setup a local OIDC provider using Ory and configure the necessary environment variables in your shell. - -2. Start the Frontend and the Backend with Docker: - - ```bash - ./deployments/scripts/launch.sh - ``` - - Or use `make` if available locally: - - ```bash - make start - ``` - - !!! note - You can also install the `Backend` and the `Frontend` using our [Helm charts](https://github.com/agntcy/identity-service/tree/main/charts). - -3. Access the Frontend UI and the Backend APIs: - - - The Backend APIs will be available at: `http://localhost:4000` for REST and `http://localhost:4001` for gRPC. - - The Frontend UI will be available at: `http://localhost:5500`. - -## Key Functionalities - -AGNTCY Identity Service is structured around two core functionalities to cater to different identity management needs: - -### Verify Identities - -This functionality is designed for users who need to confirm the legitimacy of pre-existing identities. - -- **Purpose**: To validate and integrate identities for your deployed Agentic Services. -- **Scope**: Primarily focuses on verifying identities associated with: - - **MCP Servers** (Model Context Protocol) - - **A2A Agents** (Agent-to-Agent communication entities) - - **OASF** ([Open Agentic Schema Framework](https://github.com/agntcy/oasf)) -- **Action**: Initiate the verification process by clicking the "**Verify Identity**" button. This will guide you through the steps required to authenticate your existing identities. - -### Create Identities - -This functionality empowers developers to establish new identities for their AI Agents and MCP Servers and define their associated access controls. - -- **Purpose**: To provision and manage new digital identities for Agentic Services. -- **Scope**: Allows you to add and configure identities for: - - **MCP Servers** - - **A2A Agents** - - **OASF** -- **Management**: Beyond creation, this section enables you to: - - **Manage identities**: Update, modify, or revoke existing identities. - - **Define `TBAC` Rules and Policies**: Implement Task-Based Access Control rules to govern how your Agentic Services interact and access resources, ensuring secure and compliant operations. -- **Action**: To create new identities or manage existing ones, you will typically need to "**Log In**" to your AGNTCY Identity Service account. If you are a new user, you can "**Sign Up**" to create an account and begin leveraging these features. - -## Accessing AGNTCY Identity Service - -Access to AGNTCY Identity Service's full suite of features, particularly for creating and managing identities, requires user authentication. - -- **Log In**: If you already have an account, use the "**Log In**" option to access your dashboard and manage your Agentic Service identities. -- **Sign Up**: If you are new to AGNTCY Identity Service, select "**Sign Up**" to create a new account and begin your journey with secure identity management. - -For more in-depth instructions, tutorials, and advanced configurations, please navigate through the specific sections of our comprehensive documentation. diff --git a/docs/identity/identity_service_api.md b/docs/identity/identity_service_api.md deleted file mode 100644 index 5f8bea7..0000000 --- a/docs/identity/identity_service_api.md +++ /dev/null @@ -1,3 +0,0 @@ -# Identity Service API Reference - - diff --git a/docs/identity/identity_service_api_access.md b/docs/identity/identity_service_api_access.md deleted file mode 100644 index 2bfa8c9..0000000 --- a/docs/identity/identity_service_api_access.md +++ /dev/null @@ -1,35 +0,0 @@ -# API Access - -!!! info "Endpoints" - - - **REST API Endpoint**: https://api.agent-identity.outshift.com - - **gRPC API Endpoint**: api.grpc.agent-identity.outshift.com - - **UI Endpoint**: https://agent-identity.outshift.com - -Welcome to the API Access documentation for Agent Identity. This section provides detailed information on how to interact with the Agent Identity API, including authentication, endpoints, and usage examples. - -## Organization API Key - -The Organization API Key is used to authenticate requests made to the Agent Identity API. This key is essential for accessing protected resources and performing actions on behalf of your organization. - -You can obtain your Organization API Key from the Agent Identity settings page. Ensure that you keep this key secure and do not expose it in public repositories or client-side code. - -![Organization API Key](../assets/identity/identity_service/tenant-api-key.png) - -## Agentic Service API Key - -The Agentic Service API Key is used to authenticate requests made by your application to the Agent Identity API. This key is specific to your application and should be included in the headers of your API requests or in the Python SDK configuration. - -You can generate an Agentic Service API Key from the Agent Identity settings page. Similar to the Organization API Key, ensure that this key is kept secure and not exposed in public repositories or client-side code. - -![Agentic Service API Key](../assets/identity/identity_service/app-api-key.png) - -## Protodocs - -The Protodocs definitions for the Identity Service API can be accessed [here](https://github.com/agntcy/identity-service/tree/main/docs/protodocs/agntcy/identity/service). - -## OpenAPI Client - -The OpenAPI Client provides a way to interact with the Agent Identity API using standard HTTP requests. You can use any HTTP client library to make requests to the API endpoints defined in the OpenAPI specification. - -The OpenAPI specification for the Agent Identity API can be found [here](./identity_service_api.md). diff --git a/docs/identity/identity_service_contributing.md b/docs/identity/identity_service_contributing.md deleted file mode 100644 index 5ba4a9b..0000000 --- a/docs/identity/identity_service_contributing.md +++ /dev/null @@ -1,571 +0,0 @@ -# Contributing Guide - -This guide provides an overview of the contributing process for the Agent Identity Service backend. - -# Agent Identity Service Backend Architecture - -The overall Agent Identity Service backend architecture is shown in this diagram: - -```mermaid -block - block:presentation:4 - columns 1 - pres_label["Presentation"] - Main gRPC HTTP - end - portArrow<[" "]>(right) - block:application:4 - columns 1 - app_label["Application"] - Services["Application Services"] - Domain["Domain Models"] - Repositories - end - adapterArrow<[" "]>(left) - block:infrastructure:4 - columns 1 - infrastructure_label["Infrastructure"] - Persistence - Identity["Identity Node"] - IAM["IAM & IdP"] - end - style pres_label fill:none,stroke:none - style app_label fill:none,stroke:none - style infrastructure_label fill:none,stroke:none -``` - -**Note:** The arrows indicate the dependency direction between the layers. - -The purpose of each layer is described as below: - -- **Presentation Layer:** Responsible for bootstrapping the system, implementing the gRPC services, delegating requests to the Application layer and sending responses back to clients. The main packages of this layer: - - [`cmd/bff`](https://github.com/agntcy/identity-service/tree/main/backend/cmd/bff) - - [`internal/bff/grpc`](https://github.com/agntcy/identity-service/tree/main/backend/internal/bff/grpc). -- **Application Layer:** Orchestrates and implements the core business logic of the system. Application Services within this layer are responsible for enforcing business rules, processing validations, and coordinating workflows involving domain models, persistence and external systems. -The main packages of this layer: - - [`internal/bff`](https://github.com/agntcy/identity-service/tree/main/backend/internal/bff): Implements the Application Services. - - [`internal/core`](https://github.com/agntcy/identity-service/tree/main/backend/internal/core): Implements the domain models and rest of the layer. -- **Infrasturcture Layer:** Responsible for managing technical concerns such as data persistence (repository implementation) and integration with external systems (Identity Node, IAM, IdPs, etc.). The main packages of this layer: - - [`internal/core`](https://github.com/agntcy/identity-service/tree/main/backend/internal/core): each domain contains the concrete implementation of the repositories and/or the adapters for the external systems. - - [`internal/pkg`](https://github.com/agntcy/identity-service/tree/main/backend/internal/pkg): part of this package contains adapters for external systems that are not associated with a specific domain, such as [`internal/pkg/vault`](https://github.com/agntcy/identity-service/tree/main/backend/internal/pkg/vault). - -## Bootstrapping - -The [`main()`](https://github.com/agntcy/identity-service/blob/main/backend/cmd/bff/main.go) function is responsible for initializing and running all components required by the backend. Its primary tasks include: - -1. Loading configuration -2. Establishing a database connection -3. Running database migrations -4. Initializing Application Services and injecting their dependencies -5. Starting the gRPC and HTTP servers - -### Configuration - -The backend is configured using environment variables. Additionally, any values specified in the `.env` file are automatically loaded as environment variables at startup (a sample can be found here [`cmd/bff/.env.sample`](https://github.com/agntcy/identity-service/blob/main/backend/cmd/bff/.env.sample)). -All configuration options for the system are defined in the `Configuration` struct located in [`cmd/bff/configuration.go`](https://github.com/agntcy/identity-service/blob/main/backend/cmd/bff/configuration.go). - -### Dependency injection - -In Agent Identity Service backend, the dependency injection is performed manually using [constructors](https://go.dev/doc/effective_go#composite_literals). Components depend on each other using interfaces rather than concrete implementation structs. - -```golang -type ServiceA interface {} -type ServiceB interface {} - -type serviceAImpl struct{} -func NewServiceA() ServiceA { - return &serviceAImpl{} -} - -type serviceBImpl struct{ - srvA ServiceA -} -func NewServiceB(srvA ServiceA) ServiceB { - return &serviceBImpl{srvA} -} - -sA := NewServiceA() -sB := NewServiceB(sA) -``` - -## Dependency management - -[Go modules](https://pkg.go.dev/cmd/go#hdr-Modules__module_versions__and_more) are used to manage dependencies on external packages. - -Adding a new dependency or updating an existing one is done with the `go get` command. - -After importing the dependency in the code, tidy up the `go.mod` and `go.sum` files using the `go mod tidy` command. - -> [!IMPORTANT] -> Avoid using unmaintained libraries as dependencies. - -# Services - -This document provides guidance on working with domain models, application services, and gRPC services. - -## Domain packages - -The Agent Identity Service is organized into multiple domains, with each domain defined as a separate package within [`internal/core`](https://github.com/agntcy/identity-service/tree/main/backend/internal/core). - -### Structure - -A typical domain package is structured in the following way: - -```text -domain/ -├── mocks/ -├── postgres/ -│ ├── models.go -│ └── repository.go -├── types/ -│ └── types.go -├── repository.go -└── .go -``` - -### Domain models - -Domain models are conceptual representations of the key entities, behaviors, and rules within a specific domain. They serve as the core components and building blocks of the Agent Identity Service. - -Most domains include one or more of these models, which are defined in a `types` package within each domain. - -#### Enums - -Enums in the project are defined using a custom `int` type along with a set of constants. - -For example: - -```go -type AppType int - -const ( - APP_TYPE_UNSPECIFIED AppType = iota - APP_TYPE_AGENT_A2A - APP_TYPE_AGENT_OASF - APP_TYPE_MCP_SERVER -) -``` - -A string representation for each enum value should also be generated using the [stringer](https://pkg.go.dev/golang.org/x/tools/cmd/stringer) tool. To do this, add a `//go:generate stringer` directive at the top of the file containing the enums, and run the generation with the `go generate` command. - -For example: - -```go -// Copyright 2026 AGNTCY Contributors (https://github.com/agntcy) -// SPDX-License-Identifier: Apache-2.0 - -//go:generate stringer -type=AppType -//go:generate stringer -type=AppStats - -package types - -... -``` - -#### Protobuf messages - -When domain models are defined in `types/types.go`, running `make generate_proto` will generate corresponding protobuf messages for these models. - -The generated protobuf messages serve as input and output objects for the presentation layer. - -The generation process is handled by the script [`scripts/proto/docker/run.sh`](https://github.com/agntcy/identity-service/blob/main/scripts/proto/docker/run.sh). -It utilizes Kubernetes' [`go-to-protobuf`](https://github.com/kubernetes/code-generator/tree/master/cmd/go-to-protobuf) to convert Go structs to protobuf messages -and a custom tool [`proto-enum-generator`](https://github.com/agntcy/identity-service/tree/main/scripts/proto/proto-enum-generator) to convert Go enums to protobuf enums. - -## Application services - -The entrypoint to the Application layer is the [application services](https://github.com/agntcy/identity-service/tree/main/backend/internal/bff). They are responsible for enforcing business rules, processing validations, and coordinating workflows involving domain models, persistence and external systems. - -Application services are defined as interfaces and implemented by structs. These services do not interact or reference each other directly, with the exception of the `NotificationService`. Each service can leverage multiple domain models to accomplish its tasks. - -Business logic that is shared across different application services should be encapsulated and exposed through domain models or domain services. - -## gRPC services - -gRPC services are defined as protobuf services in [`backend/api/spec/proto/agntcy/identity/service/v1alpha1`](https://github.com/agntcy/identity-service/tree/main/backend/api/spec/proto/agntcy/identity/service/v1alpha1). -These protobuf service definitions establish the API contract for the Agent Identity Service, supporting both gRPC and HTTP interfaces. - -Running `make generate_proto` generates both the server code and the Go interfaces for these protobuf services. The implementations of these interfaces are located in [`internal/bff/grpc`](https://github.com/agntcy/identity-service/tree/main/backend/internal/bff/grpc). - -### Registering gRPC services - -After implementing a gRPC service, it must be registered as a handler with both the gRPC server and the HTTP server. -This is achieved by providing an instance of the concrete implementation of the gRPC interface to the `identity_service_api.GrpcServiceRegister` struct in the `initializeServices()` function within [`main.go`](https://github.com/agntcy/identity-service/blob/main/backend/cmd/bff/main.go). - -For example: - -```go -... -register := identity_service_api.GrpcServiceRegister{ - AppServiceServer: bffgrpc.NewAppService(appSrv, badgeSrv), - SettingsServiceServer: bffgrpc.NewSettingsService(settingsSrv), - ... - MyNewService: bffgrpc.NewMyNewService(), -} -... -``` - -# Errors - -The Agent Identity Service deals with two types of errors: - -- **Domain errors:** arise when a business rule, validation or use case is violated. -These errors are represented by the custom type [`DomainError`](https://github.com/agntcy/identity-service/blob/main/backend/internal/pkg/errutil/error.go), which extends Go's `error` interface to provide additional context about the violation. -The resulting error messages are also designed to be displayed on the frontend. -- **Technical errors**: occur due to issues outside the domain logic, such as database failures or system malfunctions, that affect the system's ability to function. -These errors are returned as standard Go errors created using the functions from Go's [errors](https://pkg.go.dev/errors) or [fmt](https://pkg.go.dev/fmt) packages (e.g., `fmt.Errorf(...)`, `errors.New(...)`). - -## Domain errors - -Domain errors are returned from Application Services and Domain Services only if a business logic is violated. -The [`internal/pkg/errutil`](https://github.com/agntcy/identity-service/blob/main/backend/internal/pkg/errutil/error.go) package provides helper functions for creating `DomainError` instances based on specific reasons, for example: - -- `errutil.NotFound(id, format string, args ...any)` -- `errutil.ValidationFailed(id, format string, args ...any)` -- `errutil.InvalidRequest(id, format string, args ...any)` -- `errutil.Unauthorized(id, format string, args ...any)` - -The `id` provides a way to identify the error without relying on the message, and also enables custom error messages to be displayed on the frontend. -Throughout the codebase, the `id` is constructed as `.`, where the `` corresponds to either the Application Service name or the directory name of the domain. - -> For more information about the Application layer and its packages, see the [architecture](./identity_service_contributing.md#agent-identity-service-backend-architecture) section. - -In case a technical error must be returned, it should be wrapped to provide additional context and traceability: - -```go -return fmt.Errorf("...: %w", ..., err) -``` - -> For more information about error wrapping, see Go's [`errors`](https://pkg.go.dev/errors#pkg-overview) package. - -## Error handling in gRPC services - -Since only two types of errors are returned from the Application layer, error handling in the gRPC services is handled as follows: - -- If the error is a *domain error*, a [`status.Status`](https://pkg.go.dev/google.golang.org/grpc/status#Status) is returned with a code corresponding to the `DomainError.Reason` field. -- If the error is a *technical error*, it is forwarded as-is. -Later, a custom gRPC interceptor `ErrorInterceptor` logs the error and returns a [`status.Status`](https://pkg.go.dev/google.golang.org/grpc/status#Status) with an `Internal` code as a response. - -The helper function [`grpcutil.Error()`](https://github.com/agntcy/identity-service/blob/main/backend/internal/pkg/grpcutil/errors.go) encapsulates the error handling process described above and can be called from gRPC services, as shown in the following example: - -```go -func (s *appGrpcService) CreateApp( - ctx context.Context, - req *identity_service_sdk_go.CreateAppRequest, -) (*identity_service_sdk_go.App, error) { - createdApp, err := s.appSrv.CreateApp(ctx, ...) - if err != nil { - return nil, grpcutil.Error(err) - } - - return converters.FromApp(createdApp), nil -} -``` - -# Logging - -Logging in the Agent Identity Service is done using [logrus](https://github.com/sirupsen/logrus). -Logs are forwarded to `stdout` or `stderr` depending on their log level. Logs are structured, they can have key-value pairs which makes querying and processing logs easier and reliable. - -## Usage - -Import the [`pkg/log`](https://github.com/agntcy/identity-service/blob/main/backend/pkg/log/log.go) package to use one of the following functions to write logs: - -```go -func Debug(args ...any) -func Info(args ...any) -func Warn(args ...any) -func Error(args ...any) -``` - -For example: - -```go -log.Info("Sharing some information") - -log.Error("Oh no! ", err) -``` - -It is encouraged to exploit structured logging as much as possible, to add fields into the log simply use the `WithFields` function found in the same package: - -```go -log.WithFields(logrus.Fields{ - "key1": "value", - "key2": 123, -}).Info("Howdy, folks!") - -log.WithFields(logrus.Fields{"err": err}).Error("Oh no!") -``` - -Adding an error as a field can be done using the `WithError` function instead: - -```go -log.WithError(err).Error("Oh no!") -``` - -## Contextual logging - -For better and rich logs, it is highly recommended to use a contextual logger which enriches the log with additional key-value pairs attached to `context.Context` (e.g, `tenantID`, `appID`, `requestID`, http & gRPC requests). - -For example: - -```go -log.FromContext(ctx).Info("This log logs the context as well!") -``` - -## Log levels - -What log level to use? - -- **`log.Debug`:** Useful for times where verbosity is needed and for logs of high frequency, such as the ones used for debugging. -- **`log.Info`:** Useful for giving a steady state information about the service and important log messages. -- **`log.Warn`:** Logs that indicate a potential issue or a weird state but doesn't prevent the system from functioning. If used, it may require attention. -- **`log.Error`:** Error logs indicating unexpected behaviours occurred somewhere in the code, most of these errors are not handled. - -## Error logging - -In case an unexpected error happened and the execution flow needs to be stopped, **do not** log the error, there is a global gRPC interceptor `ErrorInterceptor` that catches these errors and logs them. -This prevents errors from being logged more than once, which facilitates tracing. - -## Formats - -The Agent Identity Service supports two logging formats based on the runtime environment: **Text logging format** and **JSON logging format**. - -### Text logging format - -> **Enabled for development environment (`GO_ENV=development`)** - -Each log is written in text format, can take multiple lines, and human-readable. - -Example: - -```text -INFO[2025-10-06T11:01:49+02:00] Starting in env:development -``` - -```text -DEBUG[2025-10-06T10:45:13+02:00] Creating badge with claims full_method=/agntcy.identity.service.v1alpha1.BadgeService/IssueBadge organization_id=6064781c-9522-401f-8edc-22ccba81a623 request=map[...] request_id=9c216d4f-f0f7-4648-9367-957dc332ed7e tenant_id=0d29df10-801e-4296-a521-8a6e5bcb10e5 -``` - -### JSON logging format - -> **Enabled for non development environment, such as production (`GO_ENV=production`)** - -Each log is written in a JSON format, which is more efficient for parsing and to be fed to observability platforms such as Grafana. - -Example: - -```json -{"level":"info","msg":"Starting in env:production","time":"2025-10-06T10:54:13+02:00"} -``` - -```json -{"full_method":"/agntcy.identity.service.v1alpha1.BadgeService/IssueBadge","level":"debug","msg":"Creating badge with claims: ...}","organization_id":"6064781c-9522-401f-8edc-22ccba81a623","request":{},"request_id":"2b35e522-47cc-4d8f-a50f-03b07f7070e8","tenant_id":"0d29df10-801e-4296-a521-8a6e5bcb10e5","time":"2025-10-06T10:58:00+02:00"} - -``` - -# Identity Context - -One of the most useful features of Go is the `Context` type, it provides a mechanism to control the lifecycle, cancellation, and propagation of request-scoped metadata across the different components of an application. - -In the Agent Identity Service, the `Context` is used to carry user and tenant information related to each gRPC request, such as `TenantID`, `AppID` and `UserID`. -The [`internal/pkg/context`](https://github.com/agntcy/identity-service/tree/main/backend/internal/pkg/context) package provides functions to set and get this information from the context. - -The context is propagated to the application and domain services, as well as to the repositories and any other components that rely on user and tenant metadata. -For example, repositories use the tenant ID from the context to filter data using the custom GORM scope [`BelongsToTenant`](https://github.com/agntcy/identity-service/blob/main/backend/internal/pkg/gormutil/scopes.go), -while HTTP clients can use the context to cancel requests if the user closes the connection to the server. - -The population of the `Context` with user and tenant metadata is managed by the `AuthInterceptor` gRPC interceptor. - -# Database - -The Agent Identity Service uses a database to persist all core entities, such as apps, issued badges, issuer settings, etc. - -The following technologies are used in the persistence layer: - -- [PostgreSQL](https://www.postgresql.org/) is used as the database engine. -- [GORM](https://gorm.io/) is used as an ORM library to simplify database interactions by mapping Go structs to database tables. For more information about GORM, refer to the [documentation](https://gorm.io/docs/index.html). - -The use of GORM or any database technology APIs is confined to the implementation of repositories. Domain and Application services interact with the persistence layer exclusively through repository interfaces. - -Each domain within [`internal/core`](https://github.com/agntcy/identity-service/tree/main/backend/internal/core) that requires persistence includes its implementation in a package named after the database engine, `postgres` in our case. - -## Declaring models - -Database models are defined as structs, with each model typically mapped to a single table. However, a model can be represented by multiple structs depending on the case. - -Most models include an ID field of type `uuid.UUID` and a `TenantID` field of type `string`, with `TenantID` being indexed in the database. Additionally, metadata fields such as `CreatedAt`, `UpdatedAt` and `DeletedAt` are included in these models. - -Database models are created using factory functions that construct an instance from a domain model. Each database model also provides a method to convert its instance back into the corresponding domain model. - -## Migrations - -Mapping database models to tables is managed through migrations. In Agent Identity Service, auto-migrations are enabled. - -To add a new database model to the migrations, update `cmd/bff/main.go` by passing an empty instance of the model to the `dbContext.AutoMigrate()` method call. - -# Testing - -At this stage, the Agent Identity Service includes only unit tests. - -Only pull requests with passing unit tests are merged. - -## Unit tests - -- Required for every file where unit testing is practical. -- Place the test file next to the file being tested. - - If no private functions/methods are being tested, add the suffix `_test` to the package name for the test (e.g., `auth_test`). - - Otherwise, use the same package name as the file under test. -- Avoid introducing infrastructure dependencies. -- One test function per function/method. - - Prefer [table-driven tests](https://go.dev/wiki/TableDrivenTests) for multiple inputs or scenarios. - - If table-driven tests aren't suitable: - - Create one test function per test case, or - - Use [`T.Run()`](https://pkg.go.dev/testing#T.Run) to create subtests within the same function. -- Arrange the body of the test using the "*Arrange -> Act -> Assert*" pattern: - - **Arrange** your objects, data, and set them up as necessary. - - **Act** on the code under test by executing it. - - **Assert** the result based on what is expected. - - ```go - func TestAppService_CreateApp_should_succeed(t *testing.T) { - t.Parallel() - - // Arrange - var ( - app apptypes.App - req identity_service_sdk_go.CreateAppRequest - ) - - _ = gofakeit.Struct(&app) - _ = gofakeit.Struct(&req) - - appSrv := bffmocks.NewAppService(t) - appSrv.EXPECT().CreateApp(t.Context(), mock.Anything).Return(&app, nil) - - sut := grpc.NewAppService(appSrv, nil) // sut: system under test - - // Act - actual, err := sut.CreateApp(t.Context(), &req) - - // Assert - assert.NoError(t, err) - assert.NotNil(t, actual) - } - ``` - -- Append `t.Parallel()` to the test when tests can safely run concurrently. - -### Naming convention - -#### Test functions with a single scenario - -When a test function covers a single scenario, its name should be composed of four parts: - -1. **Struct name** - *(include only if testing a struct method; omit for standalone functions)*. -2. **Function/method name** - The specific function or method being tested. -3. **Expected behavior** - The outcome expected when the scenario is invoked, prefer the use of the verb `should`. -4. **Test scenario** - The specific condition or context in which the function/method is being tested. - -For example: - -```go -// Testing a method -func TestAppService_CreateApp_should_return_err_when_idp_type_is_invalid(t *testing.T) - -// Testing a function -func TestVerify_should_pass_when_jwt_has_invalid_signature(t *testing.T) -``` - -#### Test functions with subtests - -When a test function contains multiple subtests, the function name should include only: - -1. **Struct name + method name** (if testing a method), or -2. **Function name** (if testing a standalone function). - -For example: - -```go -// Testing a method -func TestSettingsService_SetApiKey(t *testing.T) - -// Testing a function -func TestIsDomainError(t *testing.T) -``` - -#### Subtest naming - -Subtest names should be written as a clear, descriptive sentence that explains both the scenario being tested and the expected behavior. - -For example: - -```go -func TestSettingsService_SetApiKey(t *testing.T) { - t.Run("should set a API key for a tenant when it doesn't have one yet", func(t *testing.T) { - // ... - }) - - t.Run("should revoke an existing API key for the tenant before creating a new one", func(t *testing.T) { - // ... - }) -} -``` - -### Assertion - -All test assertions are done using the `testify` library. For more information, see the official [documentation](https://github.com/stretchr/testify). - -### Mocks - -[Mocks](https://en.wikipedia.org/wiki/Mock_object) are generated using the [`mockery`](https://vektra.github.io/mockery/latest/) tool. -To specify which interfaces should be mocked, list them explicitly in the [`.mockery.yaml`](https://github.com/agntcy/identity-service/blob/main/backend/.mockery.yaml) configuration file. - -Running `make generate_mocks` will generate all the mocks for the configured interfaces. -By default, each mock object is placed in a `mocks` package within the package of its corresponding interface. - -Under the hood, the mocks generated by `mockery` utilize the `mock` package of `github.com/stretchr/testify`. - -Usage example: - -```go -import ( - policymocks "github.com/agntcy/identity-service/internal/core/policy/mocks" - // ... -) - -func TestSomeTest(t *testing.T) { - // ... - - policyRepo := policymocks.NewPolicyRepository(t) - policyRepo.EXPECT().GetByID(ctx, policy.ID).Return(policy, nil) - - // .. -} -``` - -### Run all unit tests - -You can run the tests with `go test`. - -```sh -cd backend - -# Run all unit tests -go test ./... -``` - -Since the tests may run concurrently, run the tests multiple times to make sure no race condition is affecting the results: - -```sh -# Run each test 5 times -go test ./... -count 5 -``` - -### Test coverage - -You can run coverage analysis using the `-cover` argument: - -```sh -go test ./... -cover -``` diff --git a/docs/identity/identity_service_development.md b/docs/identity/identity_service_development.md deleted file mode 100644 index dbdab37..0000000 --- a/docs/identity/identity_service_development.md +++ /dev/null @@ -1,363 +0,0 @@ -# Development Guide - -This guide provides an overview of the development process for integrating with the **AGNTCY Identity Service**, including issuing and verifying badges, and integrating the `TBAC` (Task-Based Access Control) system in your applications. - -Before you begin, ensure you have the necessary tools and access to the **AGNTCY Identity Service**: - -- **Access to Agent Identity**: You must have an administrator role or sufficient permissions within the Agent Identity application. -- **Development Environment**: Set up your development environment with the necessary tools, such as Python or any other programming language of your choice. -- **API Keys**: Know how to obtain your Organization API Key and Agentic Service API Keys from the Agent Identity settings page. For more details, refer to the [API Access documentation](./identity_service_api_access.md). -- **Python SDK**: Install the [Python SDK documentation](./identity_service_sdk.md) to interact with the Agent Identity API and perform operations like issuing badges. -- **Documentation**: Familiarize yourself with the [API documentation](./identity_service_api_access.md) and the [Python SDK documentation](./identity_service_sdk.md). - -!!! info "Endpoints" - - Most of the development examples are provided in Python, but you can also use other programming languages to interact with the Agent Identity API. The endpoints are accessible via REST and gRPC protocols. - To see more details about the API access, you can refer to the [API](./identity_service_api_access.md) section of the documentation. - -## Issuing and Verifying Badges - -### Issuing a Badge - -To issue a badge for your Agentic Service, you can use the Python CLI or make direct API calls. The badge is essential for the discovery of your service and allows it to be recognized within the **AGNTCY Identity Service**. - -Below is an example of how to issue a badge using the Python CLI: - -```bash -identity-cli badge create {URL} -``` - -_Replace `{URL}` with the local or internal URL of your Agentic Service._ - -- You will then be prompted to provide the **Agentic Service API Key** that was generated during the "Create Agentic Service" step. Enter the API Key when requested. - -### Verifying a Badge - -To verify badges issued by Agentic Services, you can use the Python SDK or make direct API calls. The verification process ensures that the badge is valid and can be trusted for access control. - -Below is an example of how to verify a badge using the Python SDK: - -```python -from dotenv import load_dotenv -from identityservice.sdk import IdentityServiceSdk as Sdk - -load_dotenv() - -identity_sdk = Sdk( - api_key="{YOUR_ORGANIZATION_API_KEY}" -) - -try: - print( - "Got badge: ", - identity_sdk.verify_badge( - {JOSE_ENVELOPED_BADGE} - ), - ) -except Exception as e: - print("Error verifying badge: ", e) -``` - -Replace `{YOUR_ORGANIZATION_API_KEY}` with your actual [Organization API Key](./identity_service_api_access.md#organization-api-key) and `{JOSE_ENVELOPED_BADGE}` with the JOSE enveloped badge you want to verify. - -Here is the same operation using the REST API: - -```curl -curl https://{REST_API_ENDPOINT}/badges/verify \ - --request POST \ - --header 'Content-Type: application/json' \ - --header 'X-Id-Api-Key: {YOUR_ORGANIZATION_API_KEY}' \ - --data '{ - "badge": "{JOSE_ENVELOPED_BADGE}" -}' -``` - -## Task-Based Access Control (`TBAC`) (Preview) - -The **AGNTCY Identity Service** uses Task-Based Access Control (`TBAC`) to manage access between the agentic services. `TBAC` allows you to define the tasks that can be performed by each service and the permissions required to perform those tasks. - -In order to use `TBAC` effectively, you need to follow the following steps: - -1. **Create the Agentic Services and issue badges for them.** - -- Before you can define `TBAC` policies, you need to create your Agentic Services and issue badges for them. - This process involves registering your services and ensuring they are discoverable within the **AGNTCY Identity Service** (including localhost CLI). - For detailed instructions on creating Agentic Services, refer to the [Agentic Services Documentation](./creating_identities.md). - - NOTE - - The tasks available for `TBAC` are automatically discovered from the Agentic Services when you issue the badge. - -1. **Define the `TBAC` policies and rules for your Agentic Services.** - -- To integrate `TBAC` in your application, you can use the UI to define policies and rules for your Agentic Services. Please follow the detailed instructions in the [Policies and Rules Documentation](./creating_policies.md) to set up and manage access control for your services. - -1. **Integrate `TBAC` in your application** following one of the methods below. - -### A2A Integration using the Python SDK - -IMPORTANT! - -When using the Python SDK for `TBAC`, you need to provide in your environment the following variables: - -- `IDENTITY_SERVICE_API_KEY`: Your Agentic Service API Key. You can obtain this key from the Agent Services details page. - -#### Invoke using HTTPX Auth Class - -For HTTPX-based applications, you can use the `IdentityServiceAuth` class to integrate `TBAC`. This class provides an easy way to authorize Agentic Services and manage access tokens. - -```python -# Other imports -from identityservice.auth.httpx import IdentityServiceAuth - -timeout = httpx.Timeout(connect=None, read=None, write=None, pool=None) -auth = IdentityServiceAuth() # Instantiate the auth class -async with httpx.AsyncClient( -timeout=timeout, auth=auth -) as httpx_client: - -# Do your HTTPX requests here -``` - -You can see this class fully implemented in our [Financial Agentic Service](https://github.com/agntcy/identity-service/blob/main/samples/agent/oasf/financial_assistant/currency_exchange_agent.py#L112). - -##### Authorize using the A2A Starlette/FastAPI auth middleware - -For A2A (Agent-to-Agent) based applications using Starlette, you can use the `IdentityServiceA2AAuthMiddleware` to integrate `TBAC`. This middleware automatically handles authorization for incoming requests: - -- **Define a security scheme in your A2A Card** - -```python -# Define auth scheme -AUTH_SCHEME = "IdentityServiceAuthScheme" -auth_scheme = HTTPAuthSecurityScheme( - scheme="bearer", - bearerFormat="JWT", -) - -try: - # Define the Agent Card with the security scheme - agent_card = AgentCard( - ... - securitySchemes={AUTH_SCHEME: SecurityScheme(root=auth_scheme)}, - security=[ - { - AUTH_SCHEME: ["*"], - } - ], - ) -``` - -- **Add the middleware to your A2A Starlette application** - -```python -# Other imports -from identityservice.auth.starlette import IdentityServiceA2AMiddleware - -# Start server -app = server.build() - -# Add IdentityServiceMiddleware for authentication -app.add_middleware( - IdentityServiceA2AMiddleware, # Define the middleware - agent_card=agent_card, - public_paths=["/.well-known/agent.json"], -) - -# Run the application -uvicorn.run(app, host=host, port=port) -``` - -You can see this class fully implemented in our [Currency Exchange Agentic Service](https://github.com/agntcy/identity-service/blob/main/samples/agent/a2a/currency_exchange/main.py#L91). - -### MCP Integration using the Python SDK - -IMPORTANT - -When using the Python SDK for `TBAC`, you need to provide in your environment the following variables: - -- `IDENTITY_SERVICE_API_KEY`: Your Agentic Service API Key. You can obtain this key from the Agent Services details page. - -#### MCP ClientSession using HTTPX Auth Class - -For HTTPX-based applications, you can use the `IdentityServiceAuth` class to integrate `TBAC`. This class provides an easy way to authorize Agentic Services and manage access tokens. -Below you can find an example of how to use the `IdentityServiceAuth` class with HTTPX and Langchain's `MultiServerMCPClient` MCP adapter: - -```python -# Other imports -from identityservice.auth.httpx import IdentityServiceAuth - -# Init auth -auth = IdentityServiceAuth() - -# Load tools from the MCP Server -client = MultiServerMCPClient( - { - "some_mcp_server": { - "url": mcp_server_url, - "transport": "streamable_http", - "auth": auth, # Use the IdentityServiceAuth for authorization - }, - } -) -tools = await client.get_tools() - -self.graph = create_react_agent( - tools=tools, - # Other parameters -) -``` - -You can see this class fully implemented in our [Currency Exchange Agentic Service](https://github.com/agntcy/identity-service/blob/main/samples/agent/a2a/currency_exchange/agent.py#L80). - -Another example using MCP's `ClientSession` and `streamablehttp_client`: - -```python -from mcp.client.streamable_http import streamablehttp_client -from mcp import ClientSession - -from identityservice.auth.httpx import IdentityServiceAuth - -# Init auth -auth = IdentityServiceAuth() - -async def main(): - # Connect to a streamable HTTP server using the IdentityServiceAuth for authorization - async with streamablehttp_client("example/mcp", auth=auth) as ( - read_stream, - write_stream, - _, - ): - # Create a session using the client streams - async with ClientSession(read_stream, write_stream) as session: - # Initialize the connection - await session.initialize() - - # Call a tool -``` - -##### Authorize using the MCP Starlette/FastAPI auth middleware - -For MCP(Model Context Protocol) based applications using Starlette, you can use the `IdentityServiceMCPAuthMiddleware` to integrate `TBAC`. This middleware automatically handles authorization for incoming requests: - -```python -# Other imports -from identityservice.auth.starlette import IdentityServiceMCPMiddleware - -# Start server -app = server.build() - -# Add IdentityServiceMiddleware for authentication -app.add_middleware( - IdentityServiceMCPMiddleware, # Define the middleware -) - -# Run the application -uvicorn.run(app, host=host, port=port) -``` - -You can see this class fully implemented in our [Currency Exchange MCP Server](https://github.com/agntcy/identity-service/blob/main/samples/mcp/currency_exchange/main.py#L103). - -### Standard Starlette/FastAPI auth middleware - -We also support standard Starlette or FastAPI applications, you can use the `IdentityServiceAuthMiddleware` to integrate `TBAC`. This middleware automatically handles authorization for incoming requests. - -```python -# Other imports -from starlette.applications import Starlette -from starlette.middleware import Middleware - -from identityservice.auth.starlette import IdentityServiceAuthMiddleware - -routes = ... - -middleware = [ - IdentityServiceAuthMiddleware(public_paths=[]) -] - -app = Starlette(routes=routes, middleware=middleware) -``` - -### Using a custom implementation - -The Python SDK provides two functions to integrate `TBAC` in your application: - -- `access_token`: This function authorizes an Agentic Service and returns an access token that can be used to perform tasks defined in the `TBAC` policies. Optionally, you can specify the Agentic Service ID, tool name, and user token to customize the authorization process for Zero Trust environments. - -```python -access_token( - self, - agentic_service_id: str | None = None, - tool_name: str | None = None, - user_token: str | None = None, -) -> str | None -``` - -- `authorize`: This function authorizes an Agentic Service with an access token. It requires the access token and optionally the tool name. - -```python -authorize(self, access_token: str, tool_name: str | None = None) -``` - -You can use these functions to integrate `TBAC` in your application and manage access control for your Agentic Services. - -### Using the REST APIs - -IMPORTANT -When using the REST APIs for `TBAC`, you need to replace the following variables in the code snippets: - -- `REST_API_ENDPOINT`: The endpoint of the Agent Identity REST API. This can be obtained from the [API Access documentation](./identity_service_api_access.md). -- `YOUR_AGENTIC_SERVICE_API_KEY`: Your Agentic Service API Key. You can obtain this key from the Agent Services details page. - -The REST API provides endpoints to integrate `TBAC` in your application. You can follow these steps to authorize and verify Agentic Services: - -1. **Authorize an Agentic Service** - -Below are the different endpoints you can use to authorize an Agentic Service: - -- **Perform an authorization request** - -```curl -curl https://{REST_API_ENDPOINT}/auth/authorize \ - --request POST \ - --header 'Content-Type: application/json' \ - --header 'X-Id-Api-Key: {YOUR_AGENTIC_SERVICE_API_KEY}' \ - --data '{ - "appId": "", - "toolName": "", - "userToken": "" -}' -``` - -Optionally, you can specify the `appId`, `toolName`, and `userToken` to customize the authorization process for Zero Trust environments. - -- **Exchange the authorization code for an access token** - -```curl -curl https://{REST_API_ENDPOINT}/auth/token \ - --request POST \ - --header 'Content-Type: application/json' \ - --header 'X-Id-Api-Key: {YOUR_AGENTIC_SERVICE_API_KEY}' \ - --data '{ - "authorizationCode": "{AUTHORIZATION_CODE}", -}' -``` - -where `{AUTHORIZATION_CODE}` is the code received from the authorization request. - -2 **Verify an Agentic Service** - -To verify an Agentic Service, you can use the following external authorization endpoint: - -```curl -curl https://{REST_API_ENDPOINT}/auth/ext_authz \ - --request POST \ - --header 'Content-Type: application/json' \ - --header 'X-Id-Api-Key: {YOUR_AGENTIC_SERVICE_API_KEY}' \ - --data '{ - "accessToken": "{ACCESS_TOKEN}", - "toolName": "" -}' -``` - -where `{ACCESS_TOKEN}` is the access token received from the authorization request, and `toolName` is optionally the name of the tool you want to verify. diff --git a/docs/identity/identity_service_protofiles.md b/docs/identity/identity_service_protofiles.md deleted file mode 100644 index 46db4df..0000000 --- a/docs/identity/identity_service_protofiles.md +++ /dev/null @@ -1,3 +0,0 @@ -# Identity Service Protofiles - -The Protodocs definitions for the Identity Service API can be accessed [here](https://github.com/agntcy/identity-service/tree/main/docs/protodocs/agntcy/identity/service). diff --git a/docs/identity/identity_service_sdk.md b/docs/identity/identity_service_sdk.md deleted file mode 100644 index c345757..0000000 --- a/docs/identity/identity_service_sdk.md +++ /dev/null @@ -1,68 +0,0 @@ -# Identity Service SDK - -## Python SDK - -Agent Identity offers a Python SDK package allowing developers to use it as: - -- A CLI to interact with local Agentic Services (issue a badge). See the [Agentic Service Creation](./creating_identities.md#b-service-not-accessible-from-the-internet-including-localhost-and-development-deployments) for example usage when the Agentic Service is not accessible from the internet. -- A SDK to integrate TBAC for Agentic Services in your Python applications. - -To see more details about the development integration, you can refer to the [Dev section](./identity_service_development.md) section of the documentation. - -### Installation - -To install the Python SDK for Agent Identity, you can use pip: - -```bash -pip install identity-service-sdk -``` - -### Example Usage - -#### Verifying a Badge - -Here is a basic example of how to use the Python SDK to verify a badge for an Agentic Service: - -```python -from dotenv import load_dotenv -from identityservice.sdk import IdentityServiceSdk as Sdk - -load_dotenv() - -identity_sdk = Sdk( - api_key="{YOUR_ORGANIZATION_API_KEY}" -) - -try: - print( - "Got badge: ", - identity_sdk.verify_badge( - {JOSE_ENVELOPED_BADGE} - ), - ) -except Exception as e: - print("Error verifying badge: ", e) - -``` - -#### Issuing a Badge - -Here is a basic example of how to use the Python SDK to issue a badge for an Agentic Service: - -```python -from dotenv import load_dotenv -from identityservice.sdk import IdentityServiceSdk as Sdk - -load_dotenv() - -identity_sdk = Sdk( - api_key="{YOUR_AGENTIC_SERVICE_API_KEY}", -) - -try: - identity_sdk.issue_badge("{AGENTIC_SERVICE_URL}") - - print("Badge issued successfully!") -except Exception as e: - print("Error issuing badge: ", e) -``` diff --git a/docs/identity/identity_service_settings.md b/docs/identity/identity_service_settings.md deleted file mode 100644 index 291d869..0000000 --- a/docs/identity/identity_service_settings.md +++ /dev/null @@ -1,168 +0,0 @@ -# Settings - -Welcome to the "Settings" section of the Agntcy AGNTCY Identity Service. -This section provides a comprehensive guide to managing critical configurations, including API Keys, Devices and Organizations & Users. -This documentation will help you navigate the settings interface and utilize its functionalities effectively. - -## API Keys - -The API Keys subsection allows administrators to create, view, and manage API Keys within the AGNTCY Identity Service. -These keys are essential for authenticating and securing interactions between Agentic Services and external systems. - -### Accessing API Keys - -1. **Navigate to API Keys**: - From the "Settings" menu on the left-hand navigation panel, select the "API Key" tab. - -![Access API Keys Section](../assets/identity/identity_service/Settings_APIKey_00.png) - -### Creating an API Key - -1. **Create API Key:** - If no API Keys are found, click on the "+ Create API Key" button to initiate the creation process. - ![Create API Key](../assets/identity/identity_service/Settings_APIKey_01.png) - -2. **Generate API Key:** - Upon clicking "Create API Key," the system will generate a new API Key for use. - ![Generate API Key](../assets/identity/identity_service/Settings_APIKey_02.png) - -### Managing API Keys - -1. **Viewing API Keys:** - The API Keys tab will display all existing keys. Each key can be copied for use by clicking the copy icon next to it. - ![Generate API Key](../assets/identity/identity_service/Settings_APIKey_02.png) - -2. **Refresh API Key:** - To refresh an existing API Key, click the "Refresh" button. This will generate a new key and invalidate the old one. - ![Generate API Key](../assets/identity/identity_service/Settings_APIKey_02.png) - -## Devices - -The Devices section within the AGNTCY Identity Service allows you to register and manage your personal devices for secure authentication and notification purposes. -By onboarding a device, you enable it to receive push notifications for identity approvals and enhance the overall security posture of your Agent Identity environment. -This guide details the process of adding a new device and managing existing ones. - -### Adding a New Device - -Follow these steps to register and configure your device: - -1. **Access the Devices Section:** - - - From the main dashboard, locate and click on Settings in the left-hand navigation menu. - - Within the Settings section, select Devices. - - On the Devices management page, click the + Add Device button to initiate the connection wizard. - - ![Access Devices Section](../assets/identity/identity_service/Settings_DEVICE_01.png) - -2. **Onboard Your Device:** - - - A pop-up window titled "Onboard Device" will appear, displaying a QR code. This QR code is unique to your session and is used to link your mobile device. - - ![Device popup](../assets/identity/identity_service/Settings_DEVICE_02.png) - - - Using your mobile device's camera, scan the displayed QR code. Alternatively, you can click the "or click here" link if scanning is not feasible. - - ![Mobile device scan](../assets/identity/identity_service/Settings_DEVICE_03.png) - -3. **Enable Notifications on Your Mobile Device:** - - - After scanning the QR code, your mobile device will open a web page. Initially, you might see a "Notification Status: Push Notifications Not Supported" message. - -
- ![Mobile page](../assets/identity/identity_service/Settings_DEVICE_04.png) -
- - - To enable notifications, you may need to add the AGNTCY Identity Service web page to your mobile device's home screen. Access your browser's options (e.g., the share icon on iOS) and select "Add to Home Screen". - -
- ![Mobile Device Home setting](../assets/identity/identity_service/Settings_DEVICE_05.png) -
- -
- ![Mobile Device Home setting](../assets/identity/identity_service/Settings_DEVICE_06.png) -
- - - Confirm the addition. An icon for the AGNTCY Identity Service will appear on your home screen. - - ![Mobile Device Home Addition](../assets/identity/identity_service/Settings_DEVICE_07.png) - - - Open the newly created home screen application. You will be prompted to authorize notifications. Select Authorize. - -
- ![Mobile Device notification status](../assets/identity/identity_service/Settings_DEVICE_09.png) -
- - - Upon successful authorization, the "Notification Status" on the web page should update to "Push Notifications Enabled". - -
- ![Mobile Device notification status](../assets/identity/identity_service/Settings_DEVICE_08.png) -
- -### Managing Registered Devices - -Once a device is successfully added, you can view and manage it from the Devices section in the AGNTCY Identity Service portal. - -1. **Viewing Registered Devices:** - - - The Devices dashboard will now display a list of your registered devices, including details such as the device name (e.g., "iPhone (iOS 18.5) - Safari"), User ID, and the "Created At" timestamp. - - ![Mobile Device Registered](../assets/identity/identity_service/Settings_DEVICE_10.png) - -2. **Testing Device Notifications:** - - - To ensure your device is receiving notifications correctly, you can send a test notification. - - Locate the desired device in the list and click the options menu next to its entry. - - ![Mobile Device notification testing](../assets/identity/identity_service/Settings_DEVICE_11.png) - - - From the dropdown menu, select Test. - - A test notification will be sent to your registered device, appearing as a push notification on your mobile screen. - -![Mobile Device notification testing](../assets/identity/identity_service/Settings_DEVICE_12.png) - -1. **Deleting a Device:** - - - If a device is no longer needed or you wish to remove it from your account, you can delete it. - - From the options menu next to the device entry, select Delete. - - ![Mobile Device Deletion](../assets/identity/identity_service/Settings_DEVICE_13.png) - - - Confirm the deletion when prompted. - -## Organizations & Users - -The Organizations & Users subsection provides tools for managing organizational structures and user roles within the AGNTCY Identity Service. -This section is crucial for defining user access and maintaining organizational oversight. - -### Accessing Organizations & Users - -1. **Navigate to Organizations & Users:** - From the "Settings" menu, select the "Organizations & Users" tab to view existing organizations. - - ![Access Orgs and Users Section](../assets/identity/identity_service/Settings_APIKey_00.png) - -### Creating and Managing Organizations - -1. **Create New Organization:** Click on the "+ New Organization" button to start the organization creation wizard. - - ![Create new organization](../assets/identity/identity_service/Settings_ORG_01.png) - -2. **Confirmation and Creation:** A dialog box will appear asking for confirmation to create a new organization. Click "Continue" to proceed. - - ![Create new organization](../assets/identity/identity_service/Settings_ORG_02.png) - -3. **Organization Listing:** Once created, the organization will appear in the list. You can manage existing organizations by selecting options such as "Invite," "Edit," or "Delete." - - ![Create new organization](../assets/identity/identity_service/Settings_ORG_03.png) - -### Inviting Users - -1. **Invite User:** Click on "Invite User" to open the user invitation interface. - - ![Invite user](../assets/identity/identity_service/Settings_ORG_04.png) - -2. **Enter User Details:** Provide the email of the user you wish to invite and click "Invite" to send an invitation. - - ![Invite user details](../assets/identity/identity_service/Settings_ORG_05.png) - -### Managing Users diff --git a/docs/identity/openapi.md b/docs/identity/openapi.md deleted file mode 100644 index adbefdd..0000000 --- a/docs/identity/openapi.md +++ /dev/null @@ -1,3 +0,0 @@ -# OpenAPI Reference - - diff --git a/docs/identity/vc_agent_badge.md b/docs/identity/vc_agent_badge.md deleted file mode 100644 index ce1f0f4..0000000 --- a/docs/identity/vc_agent_badge.md +++ /dev/null @@ -1,86 +0,0 @@ -# Agent Badge Examples - -## OASF Agent Badge - -The example below shows an [Agent Badge](./credentials.md#agent-badge) as a VC, represented as a JSON-LD object that contains information about the Agent, including the issuing organization, its [ID](identifiers.md#definitions), a schema definition, which in this case is an [OASF Definition](../oasf/open-agentic-schema-framework.md). Such definitions may encompass additional metadata, including locators, authentication methods, hashing methods, etc. The VC below also includes a data integrity proof within an envelope. - -```json -CREDENTIAL -{ - @context: ["https://www.w3.org/ns/credentials/v2", "https://www.w3.org/ns/credentials/examples/v2"], - id: uuid(), - type: ["VerifiableCredential", "AgentBadge"], - issuer: ORG, - validFrom: date(), - credentialSubject: { - id: "ID", - badge: OASF_JSON, - }, - credentialSchema: [{ - id: "https://schema.oasf.agntcy.org/schema/objects/agent", - type: "JsonSchema" - }], - proof: { - type: "DataIntegrityProof", - proofPurpose: "assertionMethod", - proofValue: "z58DAdFfa9SkqZMVPxAQp...jQCrfFPP2oumHKtz" - } -} - -``` - -Where: - -- `credentialSubject.id`: represents the [ID](./identifiers.md#definitions) of the Agent subject. -- `credentialSubject.badge`: adheres to the [OASF Definition](../oasf/open-agentic-schema-framework.md) schema. - -## A2A Agent Badge - -Similarly, the example below shows a second Agent Badge, using in this case another definition, that is, an [A2A Agent Card](https://a2a-protocol.org/latest/definitions/) schema. - -```json -CREDENTIAL -{ - @context: ["https://www.w3.org/ns/credentials/v2", "https://www.w3.org/ns/credentials/examples/v2"], - id: uuid(), - type: ["VerifiableCredential", "AgentBadge"], - issuer: ORG, - validFrom: date(), - credentialSubject: { - id: "ID", - badge: AGENT_CARD_JSON, - }, - credentialSchema: [{ - id: "https://github.com/google/A2A/blob/main/specification/json/a2a.json#AgentCard", - type: "JsonSchema" - }], - proof: { - type: "DataIntegrityProof", - proofPurpose: "assertionMethod", - proofValue: "y58DA4DS42D35455A32Qp...jQCrfFPP2oumHKtz" - } -} - -``` - -Where: - -- `credentialSubject.id`: represents the [ID](identifiers.md#definitions) of the Agent subject. -- `credentialSubject.badge`: adheres to the [A2A Agent Card](https://a2a-protocol.org/latest/definitions/) schema. - -The `proof` in an Agent Badge can be verified using the `assertionMethod` defined in the `ResolverMetadata` object (various `ResolverMetadata` examples are available [here](./identifier_examples.md)). - -!!! note - Multiple envelopes are supported in AGNTCY, including JSON Object Signing and Encryption (JOSE). - -## Agent Badges Accessible through a "Well-Known" Endpoint - -The Agent Badges of an agent subject can be accessed using the following well-known URL: [https://api.NODE/ID/.well-known/vcs.json](https://api.NODE/ID/.well-known/vcs.json) - -Where: - -- `ID`: represents the [ID](identifiers.md#definitions) of the Agent subject. -- `NODE`: represents a **trust anchor**, that is, an Identity Node within the AGNTCY identity system. - -!!! note - Under the well-known URL above, there could be several agents badges available from the same issuer. diff --git a/docs/identity/vc_mcp.md b/docs/identity/vc_mcp.md deleted file mode 100644 index 49c3419..0000000 --- a/docs/identity/vc_mcp.md +++ /dev/null @@ -1,51 +0,0 @@ -# MCP Server Badge Example - -## MCP Server Badge - -The example below shows an [MCP Server Badge](./credentials.md) as a VC, represented as a JSON-LD object that contains information about the MCP Server, including the issuing organization, its [ID](identifiers.md), and schema definition (e.g. an [MCP Server Definition](https://spec.identity.agntcy.org/jsonschema/agntcy/identity/core/v1alpha1/McpServer#source-) in JSON). Such a definition may encompass additional metadata, including locators, authentication methods, hashing methods, etc. The VC below also includes a data integrity proof as an embedded proof within an envelope. - -```json -CREDENTIAL -{ - @context: ["https://www.w3.org/ns/credentials/v2", "https://www.w3.org/ns/credentials/examples/v2"], - id: uuid(), - type: ["VerifiableCredential", "MCPServerBadge"], - issuer: ORG, - validFrom: date(), - credentialSubject: { - id: "ID", - badge: MCP_SERVER_JSON, - }, - credentialSchema: [{ - id: "https://spec.identity.agntcy.org/jsonschema/agntcy/identity/core/v1alpha1/McpServer#source-", - type: "JsonSchema" - }], - proof: { - type: "DataIntegrityProof", - proofPurpose: "assertionMethod", - proofValue: "x36BAcFde8KkqYABCyBCq...jQCrfFPP2oumHKtz" - } -} - -``` - -Where: - -- `credentialSubject.id`: represents the [ID](identifiers.md#definitions) of the MCP Server subject. -- `credentialSubject.badge`: adheres to the [MCP Server Definition](https://spec.identity.agntcy.org/jsonschema/agntcy/identity/core/v1alpha1/McpServer#source-) schema. - -The `proof` in an MCP Server Badge can be verified using the `assertionMethod` defined in the `ResolverMetadata` object (various `ResolverMetadata` examples are available [here](./identifier_examples.md)). - -Multiple envelopes are supported in AGNTCY, including JSON Object Signing and Encryption (JOSE). - -## MCP Server Badges accessible through a "well-known" endpoint - -The MCP Server Badges can be accessed using the following well-known URL: [https://api.NODE/ID/.well-known/vcs.json](https://api.NODE/ID/.well-known/vcs.json) - -Where: - -- `ID`: represents the [ID](identifiers.md#definitions) of the MCP Server subject. -- `NODE`: represents a **trust anchor**, that is, an Identity Node within the AGNTCY identity system. - -!!! note - Under the well-known URL above, there could be several MCP Server badges available from the same issuer. diff --git a/docs/identity/verify_identity.md b/docs/identity/verify_identity.md deleted file mode 100644 index b3a41d8..0000000 --- a/docs/identity/verify_identity.md +++ /dev/null @@ -1,54 +0,0 @@ -# Verifying an Identity - -## Using the User Interface - -The "Verify Identity" screen allows users to verify a digital badge, typically a JOSE (JSON Object Signing and Encryption) enveloped badge. This screen is designed for quick and secure verification. - -1. **Access the Screen**: Navigate to the "Verify Identity" page. - - ![Verify Identity](../assets/identity/identity_service/verify-identity.png) - -2. **Provide the Badge**: - - **Option A (File Upload)**: Click or drag and drop a JSON file (max 3MB) containing the JOSE enveloped badge into the "Details" area. - - **Option B (Text Input)**: Paste the JOSE enveloped badge string directly into the "Badge" text field. - - !!! note - We support both the JOSE enveloped badge or the full JSON badge content that you can obtain from the Agentic Services. The JOSE enveloped badge is a compact representation of the badge, while the full JSON badge contains all the details in a structured format. - -3. **Initiate Verification**: Click the "Verify" button. -4. **View Results**: The "Verification Results" section will populate with the outcome of the badge verification. - - ![Verify Identity Results](../assets/identity/identity_service/verify-identity-done.png) - -## Using the Python SDK - -To verify an identity using the Python SDK, follow these steps: - -1. **Install the Python SDK:** Ensure you have the Python SDK installed. Refer to the [SDK section](./identity_service_sdk.md) of the documentation for detailed installation instructions. - -2. **Use the following code snippet:** - - ```Python - from identityservice.sdk import IdentityServiceSdk as Sdk - - from dotenv import load_dotenv - load_dotenv() - - identity_sdk = Sdk( - api_key="{YOUR_ORGANIZATION_API_KEY}" - ) - - try: - print( - "Got badge: ", - identity_sdk.verify_badge( - "{JOSE_ENVELOPED_BADGE}" - ), - ) - except Exception as e: - print("Error verifying badge: ", e) - - ``` - - !!! note - Replace `{YOUR_ORGANIZATION_API_KEY}` with your actual [Organization API Key](./identity_service_api_access.md#organization-api-key) and `{JOSE_ENVELOPED_BADGE}` with the JOSE enveloped badge you want to verify. diff --git a/docs/oasf/.index b/docs/oasf/.index deleted file mode 100644 index bcc1c6d..0000000 --- a/docs/oasf/.index +++ /dev/null @@ -1,11 +0,0 @@ -nav: - - Open Agentic Schema Framework: open-agentic-schema-framework.md - - Open Agentic Schema Framework Server: oasf-server.md - - Open Agentic Schema Framework SDK: - - OASF SDK: oasf-sdk.md - - Decoding Service: decoding.md - - Translation Service: translation.md - - Validation Service: validation.md - - Validation Comparison: validation-comparison.md - - OASF Record Guide: agent-record-guide.md - - OASF Contribution Guide: contributing.md diff --git a/docs/oasf/agent-record-guide.md b/docs/oasf/agent-record-guide.md deleted file mode 100644 index 1521a8c..0000000 --- a/docs/oasf/agent-record-guide.md +++ /dev/null @@ -1,118 +0,0 @@ -# Creating an Agent Record - -Follow the steps below to ensure that your agent record is complete and -compliant. -The model provides a structured way to describe your record's features, -capabilities, and dependencies. - -## Basic Information - -Start by filling out the basic metadata of your record: - -- `name`: - Provide a descriptive name for your record. -- `description`: - Provide a description for your record. -- `version`: - Use semantic versioning to indicate the current version of your record. -- `schema_version`: - Provide the OASF schema version the record needs to be valid against. - Must be in semantic versioning format (see [https://semver.org/](https://semver.org/)), e.g., `"0.7.0"`. -- `authors`: - List the authors in the `Name ` format. - Replace `Your Name` and `you@example.com` with the appropriate details. Providing an email is optional. -- `created_at`: - Use RFC 3339 format to specify when the record was created. - -## Define Skills - -The skills section outlines the capabilities of the agent record. -Retrieve skill definitions from the -[OASF skills catalog](https://schema.oasf.outshift.com/skill_categories). -Each skill must include the following: - -- `name`: - The name as a unique identifier (including the skill hierarchy) of a specific - skill or capability (for example, - `natural_language_processing/natural_language_generation/text_completion`). -- `id`: - The unique identifier for the skill class. - -You can add multiple skills to your record. - -## Define Domains - -The domains section outlines the domains in which your agent record excels. -Retrieve domain definitions from the -[OASF domains catalog](https://schema.oasf.outshift.com/domain_categories). -Each domain must include the following: - -- `name`: - The name as a unique identifier (including the domain hierarchy) of a specific - domain (for example, `technology/network_operations`). -- `id`: - The unique identifier for the domain class. - -You can add multiple domains to your agent. - -## Add Locators - -The locators section provides references to the agent's source code or other -resources. -If the agent is packaged as a Docker container, provide the corresponding image -or registry URL. - -You can also provide the source code: - -```json -{ - "type": "source_code", - "url": "https://github.com/agntcy/csit/tree/main/samples/crewai/simple_crew" -} -``` - -## Specify Modules - -The modules section is critical for describing the features and operational -parameters of your record. -To ensure compatibility, you must select extensions from the -[OASF modules catalog](https://schema.oasf.outshift.com/module_categories). - -## Validate and Finalize - -Double-check that all fields are filled out accurately. -Ensure that extensions and skills are selected from the OASF schema to maintain -compatibility. -Use the [Draft-07](https://schema.oasf.outshift.com/schema/objects/record) -schema of the record or the -[validation endpoint](https://schema.oasf.outshift.com/doc/index.html#/Validation/SchemaWeb.SchemaController.validate_object) -of OASF to validate your record. -Test the agent's configuration to verify that it works as expected. - -## Example Agent Record - -Below is an example of an agent record for a tourist scheduling coordinator. - -```json -{ - "name": "Tourist Scheduling Coordinator", - "description": "Central scheduling coordinator that matches tourists with tour guides…", - "version": "2.0.0", - "schema_version": "1.0.0", - "authors": ["AGNTCY "], - "created_at": "2025-01-01T00:00:00Z", - "domains": [ - {"name": "hospitality_and_tourism/tourism_management", "id": 1505} - ], - "skills": [ - {"name": "agent_orchestration/task_decomposition", "id": 1001}, - {"name": "agent_orchestration/agent_coordination", "id": 1004}, - {"name": "agent_orchestration/multi_agent_planning", "id": 1003} - ], - "modules": [ - ], - "locators": [ - {"type": "source_code", "urls": ["https://github.com/agntcy/agentic-apps/tree/main/tourist_scheduling_system"]} - ] -} -``` diff --git a/docs/oasf/contributing.md b/docs/oasf/contributing.md deleted file mode 100644 index 9d6ae22..0000000 --- a/docs/oasf/contributing.md +++ /dev/null @@ -1,467 +0,0 @@ -# OASF Contribution Guide - -Thanks for your interest in contributing to Open Agentic Schema Framework! -Here are a few general guidelines on contributing and reporting bugs that we ask -you to review. -Following these guidelines helps to communicate that you respect the time of the -contributors managing and developing this open source project. -In return, they should reciprocate that respect in addressing your issue, -assessing changes, and helping you finalize your pull requests. -In that spirit of mutual respect, we endeavor to review incoming issues and pull -requests within 10 days, and will close any lingering issues or pull requests -after 60 days of inactivity. - -Please note that all of your interactions in the project are subject to our -[Code of Conduct](https://github.com/agntcy/oasf/blob/main/CODE_OF_CONDUCT.md). -This includes creation of issues or pull requests, commenting on issues or pull -requests, and extends to all interactions in any real-time space for example, -Slack, Discord, and others. - -This documentation presents guidelines and expected etiquette to successfully -contribute to the development of OASF Schemas and the framework itself. - -## Key OASF Terminology - -- **Field**: - A field is a unique identifier name for a piece of data contained in OASF. - Each field also designates a corresponding `data_type`. -- **Object**: - An object is a collection of contextually related fields and other objects. - It is also a `data_type` in OASF. -- **Class**: - A class is a particular set of attributes (including fields & objects) - representing metadata associated to an OASF record. - It is also a `data_type` in OASF. -- **Class Family**: - Classes are currently grouped into 3 families: - Skills, Domains, and Modules. -- **Entity**: - An entity is a more generic name for both objects and classes. -- **Attribute**: - An attribute is the more generic name for both fields and entities in OASF. - A field is a scalar attribute while an entity is a complex attribute. -- **Category:** A Category organizes classes that represent a particular domain. - -## How to Add to the Schema - -### Adding or Modifying a `class` - -1. Determine where in the taxonomy of the class family you would want to add the - new `class`. -2. Create a new file → `` inside the family and category - specific subdirectory in the - [/schema](https://github.com/agntcy/oasf/tree/main/schema) folder. - Template available - [here](https://github.com/agntcy/oasf/blob/main/schema/templates/class_name.json). -3. Define the `class` itself. -4. In case of a `module` class, make sure to overwrite the `data` attribute with - a new object containing all the module-specific attributes → - [Defining an `object`](#defining-an-object). -5. Finally, verify the changes are working as expected in your local - [oasf/server](https://github.com/agntcy/oasf/tree/main/server). - -### Adding or Modifying an `attribute` - -1. All the available `attributes` - `fields` & `objects` in OASF are and will need to be defined in the attribute dictionary, the [dictionary.json](https://github.com/agntcy/oasf/blob/main/schema/dictionary.json) file and [/objects](https://github.com/agntcy/oasf/tree/main/schema/objects) folder if defining an object. -2. Determine if a new attribute is required for your change, it might already be defined in the attribute dictionary and/or the [/objects](https://github.com/agntcy/oasf/tree/main/schema/objects) folder. -3. Before adding a new attribute, review the following OASF attribute conventions: - - - Attribute names must be a valid UTF-8 sequence. - - Attribute names must be all lowercase. - - Combine words using underscore. - - No special characters except underscore. - - Use present tense unless the attribute describes historical information. - - Use singular and plural names properly to reflect the field content. - - When attribute represents multiple entities, the attribute name should be pluralized and the value type should be an array. - - Avoid repetition of words. - - Avoid abbreviations when possible. Some exceptions can be made for well-accepted abbreviation like well known acronyms (for example, LLM, AI). - - If the attribute can only be related to a `module` class, prefix its name with the module's name. - - If the attribute is supposed to hold sensitive data such as API keys, use the `env_vars` attribute instead, which enables record publishers to mandate the presence of environment variables holding such sensitive information. - -#### Defining a `field` in the Dictionary - -To add a new field in OASF, you need to define it in the -[dictionary.json](https://github.com/agntcy/oasf/blob/main/schema/dictionary.json) -file as described below. - -Sample entry in the dictionary: - -```json -"name": { - "caption": "Name", - "description": "The name of the entity. See specific usage.", - "type": "string_t" -} -``` - -Choose a **unique** field you want to add, `name` in the example above and -populate it as described below. - -1. `caption`: Add a user-friendly name to the field. -2. `description`: Add concise description to define the attributes. - - ??? note - `field` descriptions can be overridden in the `class/object`, therefore if it’s a common field (like name, label, uid, etc.) feel free to add a generic description, specific descriptions can be added in the `class/object` definition. For example: - - A generic definition of `name` in the dictionary: - 1. `name`: `The name of the entity. See specific usage.` - - Specific description of `name` in the `record` object: - 1. `name`: `The name of the record. For example: Marketing Strategy Agent.` - -3. `type`: Review OASF data_types and ensure you utilize appropriate types - while defining new fields. - - All the available data_types can be accessed [here](https://schema.oasf.outshift.com/data_types). - - They are also accessible in your [local instance of the OASF server](http://localhost:8080/data_types). -4. `is_array`: This a boolean key:value pair that you would need to add if the field you are defining is an array. - -#### Defining an `object` - -1. All the available `objects` need to be defined as individual field entries in - the dictionary, the - [dictionary.json](https://github.com/agntcy/oasf/blob/main/schema/dictionary.json) - file and as distinct `.json` files in the - [/objects](https://github.com/agntcy/oasf/tree/main/schema/objects) folder. -2. Review existing Objects, determine if a modification of the existing object - would be sufficient or if there’s a need for a completely new object. -3. Use the template - [available here](https://github.com/agntcy/oasf/blob/main/schema/templates/object_name.json), - to get started with `.json` file definition. - An example `locator.json` object file: - ```json - { - "caption": "Record Locator", - "description": "Locators provide actual artifact locators of the data's record. For example, this can reference sources such as Helm charts, Docker images, binaries, and so on.", - "extends": "object", - "name": "locator", - "attributes": { - "type": { - "caption": "Type", - "description": "Describes the type of the release manifest pointed by its URI. Allowed values MAY be defined for common manifest types.", - "requirement": "required", - "enum": { - "unspecified": { - "caption": "Unspecified" - }, - "helm_chart": { - "caption": "Helm Chart" - }, - "docker_image": { - "caption": "Docker Image" - }, - "python_package": { - "caption": "Python Package" - }, - "source_code": { - "caption": "Source Code" - }, - "binary": { - "caption": "Binary" - } - } - }, - "url": { - "caption": "URL", - "description": "Specifies an URI from which this object MAY be downloaded. Value MUST conform to RFC 3986. Value SHOULD use the http and https schemes, as defined in RFC 7230.", - "requirement": "required" - }, - "annotations": { - "caption": "Annotations", - "description": "Additional metadata associated with the record locator.", - "requirement": "optional" - }, - "size": { - "caption": "Size", - "description": "Specifies the size of the release manifest in bytes.", - "requirement": "optional" - }, - "digest": { - "caption": "Digest", - "description": "Specifies the digest of the release manifest contents.", - "requirement": "optional" - } - } - } - ``` - -4. `caption` → Add a user-friendly name to the object. - -5. `description` → Add a concise description to define the object. - -6. `extends` → Ensure the value is `object` or an existing object, e.g. - `extension_data` (All objects in OASF must extend a base definition of - `object` or another existing object.) - -7. `name` → Add a **unique** name of the object. - `name` must match the filename of the actual `.json` file. - -8. `attributes` → Add the attributes that you want to define in the object, - 1. `requirement` → For each attribute ensure you add a requirement value. - Valid values are `optional`, `required`, `recommended` - 2. `$include` → You can include attributes from other places; to do so, - specify a virtual attribute called `$include` and give its value as the - list of files (relative to the root of the schema repository) that should - contribute their attributes to this object. - _e.g._ - ```json - "attributes": { - "$include": [ - "profiles/host.json" - ], - ... - } - ``` - 3. `reference` → If the attribute's name differs from the name of the - attribute in the dictionary, you can specify the name of the dictionary - entry here. - 4. `references` → You can add a list of web links as references to the - object. - _e.g._ - ```json - "references": [ - { - "description": "A2A Card Data Schema", - "url": "https://github.com/google-a2a/A2A/blob/main/docs/specification.md#55-agentcard-object-structure" - } - ] - ``` - -9. `constraints` → For each class you can add constraints on the attribute - requirements. - Valid constraint types are `at_least_one`, `just_one`. - _e.g._ - ```json - "constraints": { - "at_least_one": [ - "id", - "name" - ] - } - ``` - -10. `references` → You can add a list of web links as references to the object. - _e.g._ - ```json - "references": [ - { - "description": "A2A Card Data Schema", - "url": "https://github.com/google-a2a/A2A/blob/main/docs/specification.md#55-agentcard-object-structure" - } - ] - ``` - -**Note:** If you want to create an object which would act only as a base for -other objects (without it being used as an enum object), you must prefix the -object `name` and the actual `json` filename with an `_`. -The resultant object will not be visible in the -[OASF Server](https://schema.oasf.outshift.com/objects). - -Sample entry in the `dictionary.json`, - -```json -"locators": { - "caption": "Record Locators", - "description": "Locators provide actual artifact locators of the data's record. For example, this can reference sources such as helm charts, docker images, binaries, and so on.", - "type": "locator", - "is_array": true -} -``` - -Choose a unique object you want to add, `locators` in the example above and -populate it as described below. - -1. `caption`: Add a user-friendly name to the object. -2. `description`: Add a concise description to define the object. -3. `type`: Add the type of the object you are defining. -4. `is_array`: This a boolean key:value pair that you would need to add if the object you are defining is an array. -5. `is_enum`: This a boolean key:value pair that you would need to add if the attribute you are defining is a `class/object` and only the entities extending the `class/object` are accepted as a value. - -### Deprecating an Attribute - -To deprecate an attribute (`field`, `object`) follow the steps below: - -1. Create a GitHub issue, explaining why an attribute needs to be deprecated and - what the alternate solution is. -2. Utilize the following flag to allow deprecation of attributes. - This flag needs to be added a json property of the attribute that is the - subject of deprecation. - ```json - "@deprecated": { - "message": "Use the ALTERNATE_ATTRIBUTE attribute instead.", - "since": "semver" - } - ``` - -3. Example of a deprecated field - ```json - "packages": { - "@deprecated": { - "message": "Use the affected_packages attribute instead.", - "since": "1.0.0" - }, - "caption": "Software Packages", - "description": "List of vulnerable packages as identified by the security product", - "is_array": true, - "type": "package" - } - ``` - -4. Example of a deprecated object - ```json - { - "caption": "Finding", - "description": "The Finding object describes metadata related to a security finding generated by a security tool or system.", - "extends": "object", - "name": "finding", - "@deprecated": { - "message": "Use the new finding_info object.", - "since": "1.0.0" - }, - "attributes": {...} - } - ``` - -### Verifying the changes - -Contributors should verify the changes before they submit the PR, the best -method to run the test suite: - -```bash -task test -``` - -To test and verify changes with a local instance of the -[oasf/server](https://github.com/agntcy/oasf/tree/main/server), follow the -[instructions here](https://github.com/agntcy/oasf/blob/main/README.md#deploy-locally) -to deploy in a local Kind cluster, or -[here](https://github.com/agntcy/oasf/blob/main/server/README.md) to set your -own local OASF server using Elixir tooling. - -Address the errors before submitting the changes, your server run should be -completely error free. - -## OASF Extensions - -The OASF Schema can be extended by adding an extension that defines additional -attributes, objects, profiles, classes, or categories. -Extensions allow one to create vendor/customer specific schemas or augment an -existing schema to better suit their custom requirements. -Extensions can also be used to factor out non-essential schema domains keeping -the core schema succinct. -Extensions use the framework in the same way as a new schema, optionally -creating categories, profiles, or classes from the dictionary. - -As with categories and classes, extensions have unique IDs within the framework -as well as their own versioning. -The following sections provide guidelines to create extensions within OASF. - -### Reserve a UID and Name for Your Extension - -In order to reserve an ID space, and make your extension public, add a unique -identifier & a unique name for your extension in the OASF Extensions Registry -[here](https://github.com/agntcy/oasf/blob/main/schema/extensions.md). -This is done to avoid collisions with core or other extension schemas. -For example, a new sample extension would have a row in the table as follows: - -| **Caption** | **Name** | **UID** | **Notes** | -| ------------- | -------- | ------- | --------------------------------- | -| New Extension | new_ex | 123 | The development schema extensions | - -### Create Your Extension's Subdirectory - -To extend the schema, create a new subdirectory in the `extensions` directory, -and add a new `extension.json` file, which defines the extension's `name` and -`uid`. -For example: - -```json -{ - "caption": "New Extension", - "name": "new_ex", - "uid": 123, - "version": "0.0.0" -} -``` - -The extension's directory structure is the same as the top level schema -directory, and it may contain the following files and subdirectories, depending -on what type of extension is desired: - -| Name | Description | -| ------------------- | ------------------------------------------ | -| `main_skills.json` | Create it to define new skill categories. | -| `main_domains.json` | Create it to define new domain categories. | -| `main_modules.json` | Create it to define new module categories. | -| `dictionary.json` | Create it to define new attributes. | -| `skills` | Create it to define new skill classes. | -| `domains` | Create it to define new domain classes. | -| `modules` | Create it to define new module classes. | -| `objects` | Create it to define new objects. | -| `profiles` | Create it to define new profiles. | - -## Reporting Issues - -Before reporting a new issue, please ensure that the issue was not already -reported or fixed by searching through our -[issues list](https://github.com/agntcy/oasf/issues). - -When creating a new issue, please be sure to include a **title and clear -description**, as much relevant information as possible, and, if possible, a -test case. - -**If you discover a security bug, please do not report it through GitHub. -Instead, please see security procedures in -[SECURITY.md](https://github.com/agntcy/oasf/blob/main/SECURITY.md).** - -## Sending Pull Requests - -Before sending a new pull request, take a look at existing pull requests and -issues to see if the proposed change or fix has been discussed in the past, or -if the change was already implemented but not yet released. - -We expect new pull requests to include tests for any affected behavior, and, as -we follow semantic versioning, we may reserve breaking changes until the next -major version release. - -## Developer’s Certificate of Origin - -To improve tracking of who did what, we have introduced a “sign-off” procedure. -The sign-off is a line at the end of the explanation for the commit, which -certifies that you wrote it or otherwise have the right to pass it on as open -source work. -We use the Developer Certificate of Origin (see -https://developercertificate.org/) for our sign-off procedure. -You must include a sign-off in the commit message of your pull request for it to -be accepted. -The format for a sign-off is: - -```text -Signed-off-by: Random J Developer -``` - -You can use the -s when you do a git commit to simplify including a properly -formatted sign-off in your commits. -If you need to add your sign-off to a commit you have already made, you will -need to amend: - -```sh -git commit --amend --signoff -``` - -## Other Ways to Contribute - -We welcome anyone that wants to contribute to OASF to triage and reply to open -issues to help troubleshoot and fix existing bugs. -Here is what you can do: - -- Help ensure that existing issues follows the recommendations from the - _[Reporting Issues](#reporting-issues)_ section, providing feedback to the - issue's author on what might be missing. -- Review and update the existing content of our - [docs](https://docs.agntcy.org/oasf/open-agentic-schema-framework) with - up-to-date instructions and code samples. -- Review existing pull requests, and testing patches against real existing - applications that use `OASF`. -- Write a test, or add a missing test case to an existing test. - -Thanks again for your interest on contributing to `OASF`! diff --git a/docs/oasf/decoding.md b/docs/oasf/decoding.md deleted file mode 100644 index 530a48a..0000000 --- a/docs/oasf/decoding.md +++ /dev/null @@ -1,73 +0,0 @@ -# Decoding Service - -The decoding service converts between different data formats and creates OASF-compliant records. - -## `JsonToProto` - -Converts a JSON object to a Protocol Buffer Struct. - -```go -func JsonToProto(data []byte) (*structpb.Struct, error) -``` - -**Parameters:** - -* `data`: JSON data as byte array - -Returns `*structpb.Struct`: Protocol Buffer Struct representation. - -## `StructToProto` - -Converts a Go struct to a Protocol Buffer Struct. - -```go -func StructToProto(goObj any) (*structpb.Struct, error) -``` - -**Parameters:** - -* `goObj`: Any Go struct or object - -Returns `*structpb.Struct`: Protocol Buffer Struct representation. - -## `ProtoToStruct` - -Converts a Protocol Buffer Struct to a Go struct. - -```go -func ProtoToStruct[T any](obj *structpb.Struct) (*T, error) -``` - -**Parameters:** - -* `obj`: Protocol Buffer Struct to convert - -Returns `*T`: Pointer to the converted Go struct. - -## `DecodeRecord` - -Decodes a record object into a structured format based on its schema version. - -```go -func DecodeRecord(record *structpb.Struct) (*decodingv1.DecodeRecordResponse, error) -``` - -**Parameters:** - -* `record`: OASF record as Protocol Buffer Struct - -Returns `*decodingv1.DecodeRecordResponse`: decoded record response with version-specific structure. - -## `GetRecordSchemaVersion` - -Extracts the schema version from a record object. - -```go -func GetRecordSchemaVersion(record *structpb.Struct) (string, error) -``` - -**Parameters:** - -* `record`: OASF record as Protocol Buffer Struct - -Returns `string`: schema version (e.g., "0.7.0", "1.0.0"). diff --git a/docs/oasf/oasf-sdk.md b/docs/oasf/oasf-sdk.md deleted file mode 100644 index ef21978..0000000 --- a/docs/oasf/oasf-sdk.md +++ /dev/null @@ -1,38 +0,0 @@ -# OASF SDK - -The OASF SDK provides three services for working with OASF records: - -* Validation: Validate OASF records against schema specifications. -* Translation: Convert OASF records to different formats (GitHub Copilot MCP, A2A). -* Decoding: Convert between JSON, Go structs, and Protocol Buffer structures. - -The proto files can be found [here](https://buf.build/agntcy/oasf-sdk). - -## Installation - -The OASF SDK is available as a Go module and a Helm chart. - -Add the OASF SDK package to your Go project: - -```bash -go get github.com/agntcy/oasf-sdk/pkg@latest -``` - -Deploy the OASF SDK server using the provided Helm chart: - -```bash -helm install oasf-sdk ./helm/oasf-sdk -``` - -Alternatively, you can deploy the OASF SDK server using the provided Docker image: - -```bash -docker run -p 31234:31234 ghcr.io/agntcy/oasf-sdk:latest -``` - -## Usage - -The OASF SDK can be used in two ways: - -* As a Go Package you can import and use the packages directly in your Go application. -* As a gRPC Service you can deploy the server and communicate via gRPC. diff --git a/docs/oasf/oasf-server.md b/docs/oasf/oasf-server.md deleted file mode 100644 index 7ed0a26..0000000 --- a/docs/oasf/oasf-server.md +++ /dev/null @@ -1,119 +0,0 @@ -# Open Agentic Schema Framework Server - -The `server/` directory contains the Open Agentic Schema Framework (OASF) Schema -Server source code. -The schema server is an HTTP server that provides a convenient way to browse and -use the OASF schema. -The server provides also schema validation capabilities to be used during -development. - -You can access the OASF schema server, which is running the latest released -schema, at [schema.oasf.outshift.com](https://schema.oasf.outshift.com). - -The schema server can also be used locally. - -## Development - -Use `Taskfile` for all related development operations such as testing, -validating, deploying, and working with the project. - -Check the -[example.env](https://github.com/agntcy/oasf/blob/main/server/.env.sample) to -see the configuration for the operations below. - -### Prerequisites - -- [Taskfile](https://taskfile.dev/) -- [Docker](https://www.docker.com/) -- [Go](https://go.dev/) -- [yq](https://github.com/mikefarah/yq) -- [curl](https://curl.se/) -- [tar](https://www.gnu.org/software/tar/) - -Make sure Docker is installed with Buildx. - -### Clone the Repository - -```shell -git clone https://github.com/agntcy/oasf.git -``` - -### Build Artifacts - -This step will fetch all project dependencies and subsequently build all project -artifacts such as helm charts and Docker images. - -```shell -task deps -task build -``` - -### Deploy Locally - -This step will create an ephemeral Kind cluster and deploy OASF services via -Helm chart. -It also sets up port forwarding so that the services can be accessed locally. - -```shell -IMAGE_TAG=latest task build:images -task up -``` - -To access the schema server, open [`localhost:8080`](http://localhost:8080) in -your browser. - -**Note:** Any changes made to the server backend itself will require running -`task up` again. - -To set your own local OASF server using Elixir tooling, follow -[these instructions](https://github.com/agntcy/oasf/blob/main/server/README.md). - -### Hot Reload - -In order to run the server in hot-reload mode, you must first deploy the -services, and run another command to signal that the schema will be actively -updated. - -This can be achieved by starting an interactive reload session via: - -```shell -task reload -``` - -**Note:** This will only perform hot-reload for schema changes. -Reloading backend changes still requires re-running `task build && task up`. - -### Deploy Locally with Multiple Versions - -Trying out OASF locally with multiple versions is also possible, with updating -the `install/charts/oasf/values-test-versions.yaml` file with the required -versions and deploying OASF services on the ephemeral Kind cluster with those -values. - -``` -HELM_VALUES_PATH=./install/charts/oasf/values-test-versions.yaml task up -``` - -### Cleanup - -This step will handle cleanup procedure by removing resources from previous -steps, including ephemeral Kind clusters and Docker containers. - -```shell -task down -``` - -## Distribution - -### Artifacts - -See -[AGNTCY Github Registry](https://github.com/orgs/agntcy/packages?repo_name=oasf). - -### Protocol Buffer Definitions - -The `proto` directory contains the Protocol Buffer (`.proto`) files defining our -data objects and APIs. -The full proto module, generated language stubs and it's versions are hosted at -the Buf Schema Registry: -[https://buf.build/agntcy/oasf](https://buf.build/agntcy/oasf) diff --git a/docs/oasf/open-agentic-schema-framework.md b/docs/oasf/open-agentic-schema-framework.md deleted file mode 100644 index 849b616..0000000 --- a/docs/oasf/open-agentic-schema-framework.md +++ /dev/null @@ -1,78 +0,0 @@ -# Open Agentic Schema Framework - -The [Open Agentic Schema Framework (OASF)](https://schema.oasf.outshift.com/) is -a standardized schema system for defining and managing AI agent capabilities, -interactions, and metadata. -It provides a structured way to describe agent attributes, capabilities, and -relationships using attribute-based taxonomies. -The framework includes development tools, schema validation, and hot-reload -capabilities for rapid schema development, all managed through a Taskfile-based -workflow and containerized development environment. -The OASF serves as the foundation for interoperable AI agent systems, enabling -consistent definition and discovery of agent capabilities across distributed -systems. - -OASF is highly inspired from -[OCSF (Open Cybersecurity Schema Framework)](https://ocsf.io/) not only in terms of data -modeling philosophy but also in terms of implementation. -The server is a derivative work of OCSF schema server and the schema update -workflows reproduce those developed by OCSF. - -## Features - -OASF defines a set of standards for AI agent content representation that aims -to: - -- Define common data structure to facilitate content standardization, - validation, and interoperability. -- Ensure unique agent identification to address content discovery and - consumption. -- Provide extension capabilities to enable third-party features. - -## Key Concepts - -At the core of OASF is the -[record object](https://schema.oasf.outshift.com/objects/record), which serves -as the primary data structure for representing collections of information and -metadata relevant to agentic AI applications. - -OASF records can be annotated with **skills** and **domains** to enable -effective announcement and discovery across agentic systems. -Additionally, **modules** provide a flexible mechanism to extend records with -additional information in a modular and composable way, supporting a wide range -of agentic use cases. - -## Schema Expansion and Contributions - -The Open Agentic Schema Framework (OASF) is designed with extensibility in mind -and is expected to evolve to capture new use cases and capabilities. -A key area of anticipated expansion includes the definition and management of -**Skills**, **Domains** and **Modules** for AI agentic records. - -We welcome contributions from the community to help shape the future of OASF. -For detailed guidelines on how to contribute, including information on proposing -new features, reporting bugs, and submitting code, please refer to our -[contributing guide](contributing.md). - -OASF can be extended with private schema extensions, allowing you to leverage -all features of the framework, such as validation. -See the relevant section in the -[contributing guide](contributing.md#oasf-extensions) for instructions on adding -an extension to the schema. -An OASF instance with schema extensions can be hosted, allowing you to use your -own schema server for record validation. - -## Useful Links - -A convenient way to browse and use the OASF schema is through the -[Open Agentic Schema Framework Server](https://schema.oasf.outshift.com) hosted -by Outshift by Cisco. - -To deploy the server either locally or as a hosted service, see the -[server's guide](oasf-server.md) for more information. - -See [Creating an Agent Record](./agent-record-guide.md) for more -information on the Agent Record. - -The current skill set taxonomy is described in -[Taxonomy of AI Agent Skills](https://schema.oasf.outshift.com/skill_categories). diff --git a/docs/oasf/translation.md b/docs/oasf/translation.md deleted file mode 100644 index 42e535c..0000000 --- a/docs/oasf/translation.md +++ /dev/null @@ -1,73 +0,0 @@ -# Translation Service - -The translation service converts OASF records into GitHub Copilot MCP and A2A configuration structures. - -## GitHub Copilot MCP - -`RecordToGHCopilot` translates an OASF record into a GitHub Copilot MCP configuration structure. - -```go -func RecordToGHCopilot(record *structpb.Struct) (*GHCopilotMCPConfig, error) -``` - -**Returns:** - -`*GHCopilotMCPConfig`: MCP configuration for GitHub Copilot - -**GHCopilotMCPConfig Structure:** - -```go -type GHCopilotMCPConfig struct { - Servers map[string]MCPServer `json:"servers"` - Inputs []MCPInput `json:"inputs"` -} - -type MCPServer struct { - Command string `json:"command"` - Args []string `json:"args"` - Env map[string]string `json:"env"` -} - -type MCPInput struct { - ID string `json:"id"` - Type string `json:"type"` - Password bool `json:"password"` - Description string `json:"description"` -} -``` - -## A2A - -`RecordToA2A` translates an OASF record into an A2A card structure. - -```go -func RecordToA2A(record *structpb.Struct) (*A2ACard, error) -``` - -**Returns:** - -`*A2ACard`: A2A card configuration - -**A2ACard Structure:** - -```go -type A2ACard struct { - Name string `json:"name"` - Description string `json:"description"` - URL string `json:"url"` - Capabilities map[string]bool `json:"capabilities"` - DefaultInputModes []string `json:"defaultInputModes"` - DefaultOutputModes []string `json:"defaultOutputModes"` - Skills []A2ASkill `json:"skills"` -} - -type A2ASkill struct { - ID string `json:"id"` - Name string `json:"name"` - Description string `json:"description"` -} -``` - -## Example Usage - -For detailed examples, see the [OASF SDK repository](https://github.com/agntcy/oasf-sdk/blob/main/USAGE.md). diff --git a/docs/oasf/validation-comparison.md b/docs/oasf/validation-comparison.md deleted file mode 100644 index 8ce0e83..0000000 --- a/docs/oasf/validation-comparison.md +++ /dev/null @@ -1,109 +0,0 @@ -# Validation Comparison: API Validator vs JSON Schema Draft-07 - -This table compares validation outcomes between the OASF API validator and JSON Schema Draft-07 validator. - -## Legend - -- **API Error**: - API validator returns ERROR -- **API Warning**: - API validator returns WARNING -- **JSON Schema Pass**: - JSON Schema Draft-07 validation passes -- **JSON Schema Fail**: - JSON Schema Draft-07 validation fails - -## Comparison Table - -| Case | Example | API Validator | JSON Schema | Notes | -| ---------------------------------------- | --------------------------------------------------------------------- | -------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------ | -| 1. Version Incompatibility (Later) | `schema_version: "1.0.0"` when server is `0.8.0` | ERROR (`version_incompatible_later`) | PASS | JSON Schema validates format only, not semantic compatibility 1 | -| 2. Version Incompatibility (Initial Dev) | `schema_version: "0.1.0"` when server is `0.8.0` | ERROR (`version_incompatible_initial_development`) | PASS | JSON Schema validates format only, not SemVer rules 1 | -| 3. Version Incompatibility (Prerelease) | `schema_version: "0.8.0-alpha"` when server is `0.8.0` | ERROR (`version_incompatible_prerelease`) | PASS | JSON Schema validates format only, not prerelease compatibility 1 | -| 4. Version Earlier (Compatible) | `schema_version: "0.7.0"` when server is `0.8.0` | WARNING (`version_earlier`) | PASS | JSON Schema validates format only, not version comparison 1 | -| 5. Regex Pattern Mismatch | `previous_record_cid: "invalid"` (doesn't match CID regex) | WARNING (`attribute_value_regex_not_matched`) | PASS | JSON Schema may export regex but validation is lenient; API treats as warning | -| 6. Regex Pattern Mismatch (Super Type) | String doesn't match super type regex | WARNING (`attribute_value_super_type_regex_not_matched`) | PASS | Same as above for super types | -| 7. Using Base Class | `skills: [{"name": "base_skill"}]` or `skills: [{"id": 0}]` | ERROR (`base_class_used`) | FAIL | Both catch base classes (base_skill, base_domain, base_module); JSON Schema `oneOf` doesn't include parent classes | -| 8. Using Deprecated Class | Using a deprecated skill/domain/module | WARNING (`class_deprecated`) | PASS | JSON Schema doesn't track deprecation | -| 9. Using Deprecated Attribute | Using a deprecated attribute | WARNING (`attribute_deprecated`) | PASS | JSON Schema doesn't track deprecation | -| 10. Using Deprecated Object | Using a deprecated object | WARNING (`object_deprecated`) | PASS | JSON Schema doesn't track deprecation | -| 11. Using Deprecated Enum Value | Using a deprecated enum value | WARNING (`attribute_enum_value_deprecated`) | PASS | JSON Schema doesn't track deprecation | -| 12. Using Deprecated Enum Array Value | Using a deprecated enum array value | WARNING (`attribute_enum_array_value_deprecated`) | PASS | JSON Schema doesn't track deprecation | -| 13. Enum Sibling Mismatch | Enum value doesn't match sibling caption | WARNING (`attribute_enum_sibling_incorrect`) | PASS | JSON Schema doesn't validate enum siblings | -| 14. Enum Sibling Suspicious (Other) | Enum value 99 with matching sibling caption | WARNING (`attribute_enum_sibling_suspicious_other`) | PASS | JSON Schema doesn't validate enum siblings | -| 15. Missing Recommended Attribute | Required attribute missing (if `warn_on_missing_recommended` enabled) | WARNING (`attribute_recommended_missing`) | PASS | JSON Schema only validates `required`, not `recommended` | -| 16. Empty Required Array | `skills: []` when skills is required | ERROR (`attribute_required_empty`) | FAIL | JSON Schema now includes `minItems: 1` for required arrays (after fix) | -| 17. Wrong Type | `name: 123` when name should be string | ERROR (`attribute_wrong_type`) | FAIL | Both catch type mismatches | -| 18. Missing Required Attribute | Missing required `name` field | ERROR (`attribute_required_missing`) | FAIL | Both catch missing required fields | -| 19. Unknown Attribute | Extra attribute not in schema | ERROR (`attribute_unknown`) | FAIL | JSON Schema uses `additionalProperties: false` | -| 20. Unknown Enum Value | Enum value not in allowed list | ERROR (`attribute_enum_value_unknown`) | FAIL | JSON Schema uses `enum` constraint | -| 21. Unknown Enum Array Value | Enum array value not in allowed list | ERROR (`attribute_enum_array_value_unknown`) | FAIL | JSON Schema validates array items | -| 22. Value Outside Range | Number outside type's range | ERROR (`attribute_value_exceeds_range`) | FAIL | JSON Schema uses `minimum`/`maximum` | -| 23. Value Exceeds Max Length | String exceeds type's max length | ERROR (`attribute_value_exceeds_max_len`) | FAIL | JSON Schema uses `maxLength` | -| 24. Value Not in Type Values | Value not in type's allowed values list | ERROR (`attribute_value_not_in_type_values`) | FAIL | JSON Schema uses `enum` constraint | -| 25. Unknown Class ID | `id: 99999` doesn't exist | ERROR (`id_unknown`) | FAIL | JSON Schema uses `oneOf` with `const` values | -| 26. Unknown Class Name | `name: "nonexistent"` doesn't exist | ERROR (`name_unknown`) | FAIL | JSON Schema uses `oneOf` with `const` values | -| 27. ID/Name Mismatch | `id: 2` and `name: "different"` refer to different classes | ERROR (`id_name_mismatch`) | FAIL | JSON Schema `oneOf` with `const` prevents mismatches | -| 28. Constraint Failed (at_least_one) | None of the constraint fields present | ERROR (`constraint_failed`) | FAIL | JSON Schema uses `anyOf` for `at_least_one` | -| 29. Constraint Failed (just_one) | Both fields in `just_one` constraint present | ERROR (`constraint_failed`) | FAIL | JSON Schema `oneOf` pattern correctly prevents both fields from being present | -| 30. Enum Array Sibling Missing | Enum array sibling array too short | ERROR (`attribute_enum_array_sibling_missing`) | PASS | JSON Schema doesn't validate enum array siblings | -| 31. Enum Array Sibling Incorrect | Enum array sibling value doesn't match | ERROR (`attribute_enum_array_sibling_incorrect`) | PASS | JSON Schema doesn't validate enum array siblings | -| 32. Enum Object Not Matched | Enum object doesn't match any allowed objects | ERROR (`enum_object_not_matched`) | FAIL | JSON Schema uses `oneOf` for enum objects | -| 33. Unknown Profile | Profile in `metadata.profiles` doesn't exist | ERROR (`profile_unknown`) | PASS | JSON Schema validates structure but not profile existence (only for classes, not record) | -| 34. Invalid Version Format | `schema_version: "invalid"` | ERROR (`version_invalid_format`) | FAIL | JSON Schema validates semantic versioning format 1 | -| 35. Array Duplicates | Duplicate items in array (e.g., same skill twice) | ERROR (`attribute_array_duplicate`) | FAIL | Both catch duplicates; JSON Schema uses `uniqueItems: true` | -| 36. Locators Duplicate Types | Multiple locators with same type | ERROR (`attribute_locators_duplicate_type`) | FAIL | Both catch duplicate types in locators array; JSON Schema uses `uniqueItems: true` | - - -1 When using the OASF SDK, any schema version that is not explicitly supported by the decoder (currently: 0.7.0, 0.8.0, and 1.0.0) results in a failure, regardless of whether the API validator returns an error or warning. The SDK uses a switch statement that only handles specific versions, and any non-supported version returns an error: `unsupported OASF version: `. - -## Summary by Category - -### API ERROR, JSON Schema PASSES (Gaps in JSON Schema) - -1. Version incompatibility (later, initial dev, prerelease) -2. Enum array sibling validation (missing, incorrect) -3. Unknown profile (for classes) - -### API WARNING, JSON Schema PASSES (Non-blocking Issues) - -1. Version earlier (compatible) -2. Regex pattern mismatches -3. All deprecation warnings (class, attribute, object, enum values) -4. Enum sibling mismatches -5. Missing recommended attributes - -### Both FAIL (Both Catch the Issue) - -1. Empty required arrays -2. Wrong types -3. Missing required attributes -4. Unknown attributes -5. Unknown enum values -6. Value range/length violations -7. Unknown class ID/name (including unknown modules, skills, domains) -8. ID/name mismatches -9. Constraint violations (`just_one`, `at_least_one`) -10. Enum object mismatches -11. Invalid version format -12. Using base classes (base_skill, base_domain, base_module) -13. Array duplicates (duplicate items in arrays) -14. Locators duplicate types (duplicate types in locators array) - -## Key Insights - -1. JSON Schema is comprehensive for structural and value validation -2. Main gaps in JSON Schema: - - - Semantic version compatibility (only validates format) - - Enum array sibling validation - - Deprecation tracking - - Regex validation (treated as warnings, not errors) - - Profile existence (for classes) - -3. Both validators catch most structural and type validation issues, including: - - - Base classes (now errors in API validator, matching JSON Schema behavior) - - Array duplicates (both use duplicate detection) - - Locators duplicate types (both catch duplicate types in locators array) - - Unknown classes (modules, skills, domains - all treated the same way) diff --git a/docs/oasf/validation.md b/docs/oasf/validation.md deleted file mode 100644 index ea00dda..0000000 --- a/docs/oasf/validation.md +++ /dev/null @@ -1,67 +0,0 @@ -# Validation Service - -The validation service validates OASF records against an OASF schema server via API validation. -The validation service supports the following features: - -- Validates records against any OASF schema server URL. -- Returns detailed validation errors and warnings separately. -- Automatic schema version detection from records. -- Warnings do not affect validation result (only errors cause validation to fail). - -## Initialization - -Create a validator instance by providing a schema URL: - -```go -validator, err := validator.New("https://schema.oasf.outshift.com") -if err != nil { - // Handle error -} -``` - -The schema URL is required and must be provided when creating the validator. -The validator uses this URL for all validation operations. - -## Validation - -Use `ValidateRecord` to validate a record against the configured schema URL. - -**Parameters:** - -- `ctx`: Context for cancellation and timeout control -- `record`: The OASF record to validate (as a Protocol Buffer Struct) - -**Returns:** - -- `bool`: Whether the record is valid (true if no errors, false if errors present) -- `[]string`: List of validation error messages (empty if valid) -- `[]string`: List of validation warning messages (may be non-empty even if valid) -- `error`: Any error that occurred during validation - -!!! note - Warnings do not affect the validation result. A record is considered valid if there are no errors, regardless of whether warnings are present. - -## Validation Response - -The validation response includes errors and warnings. - -Errors are critical validation failures that must be fixed. If any errors are present, the record is invalid. Warnings are non-critical issues or deprecation notices. Warnings do not affect the validation result. - -Error messages include the following: - -- Clear descriptions of what failed validation. -- Attribute paths (e.g., `data.servers[0]`). -- Constraint details for `constraint_failed` errors. - -Warning messages include the following: - -- Descriptions of non-critical issues. -- Attribute paths where applicable. - -## Example Usage - -For detailed examples, see the [OASF SDK repository](https://github.com/agntcy/oasf-sdk/blob/main/USAGE.md). - -## Validation Comparison - -For a detailed comparison between the API validator and JSON Schema Draft-07 validation, including differences in error handling, warnings, and validation coverage, see the [Validation Comparison](validation-comparison.md) page. diff --git a/docs/slim/.index b/docs/slim/.index deleted file mode 100644 index ed920b6..0000000 --- a/docs/slim/.index +++ /dev/null @@ -1,21 +0,0 @@ -nav: - - Overview: overview.md - - Getting Started with SLIM: - - Getting Started: slim-howto.md - - Configuration Reference: slim-data-plane-config.md - - SLIM Controller: slim-controller.md - - SLIM Controller Reference: slim-controller-reference.md - - SLIM Messaging Layer: - - Overview: slim-data-plane.md - - SLIM Sessions: slim-session.md - - SLIM Authentication: slim-authentication.md - - SLIM Groups: - - Group Creation and Management: slim-group.md - - Group Communication Tutorial: slim-group-tutorial.md - - SLIM Integrations: - - SLIMRPC: - - Overview: slim-rpc.md - - Protoc Plugin: slim-slimrpc-compiler.md - - SLIM A2A: slim-a2a.md - - MCP over SLIM: slim-mcp.md - - OpenTelemetry over SLIM: slim-otel.md diff --git a/docs/slim/overview.md b/docs/slim/overview.md deleted file mode 100644 index a7abd26..0000000 --- a/docs/slim/overview.md +++ /dev/null @@ -1,123 +0,0 @@ -# Overview - -Secure Low-Latency Interactive Messaging (SLIM) is a secure, scalable, and developer-friendly messaging framework that -provides the transport layer for agent communication protocols like A2A. While -A2A defines *what* agents say (message formats, task semantics, coordination -patterns), SLIM defines *how* these messages are securely delivered across -distributed networks. - -At its core, SLIM combines: - -- **gRPC's performance and reliability** — Built on HTTP/2 for efficient, - multiplexed transport -- **Messaging capabilities** — Native support for channels and group - communication -- **End-to-end encryption** — Using Message Layer Security (MLS) protocol -- **Native RPC support** — SRPC (SLIM RPC) for request-response patterns - alongside messaging -- **Distributed architecture** — Separate control and data planes for - scalability and management -- **Protocol flexibility** — Transport layer for A2A, MCP, and custom agent - protocols - -SLIM enables AI agents to communicate securely whether they're running in a data -center, in a browser, on mobile devices, or across organizational boundaries — -all while maintaining low latencies and strong security guarantees. - -## SLIM Components - -SLIM is composed of two main components that work together to provide secure, -scalable messaging infrastructure: - -- [SLIM Messaging Layer](./slim-data-plane.md): The data plane component - that handles message routing, delivery, and secure communication between - applications. It consists of two layers: the session layer that provides - end-to-end encryption (using the MLS protocol) and reliable message delivery, - and the data plane that enables efficient message distribution across the - network. - -- [SLIM Controller](./slim-controller.md): The control plane component that - manages SLIM node configurations, monitors the network, and provides a unified - interface for administering the messaging infrastructure. It enables - centralized management of routes, connections, and node deployments. - -### Architecture Overview - -The following diagram illustrates how SLIM components are distributed across -applications and intermediate routing nodes: - -```mermaid -graph TB - subgraph "Application Node (Agent A)" - A1[Session Layer 1] - A2[Session Layer 2] - A3[Data Plane Client] - A1 --> A3 - A2 --> A3 - end - - subgraph "Intermediate SLIM Node 1" - I1[Data Plane] - end - - subgraph "Intermediate SLIM Node 2" - I2[Data Plane] - end - - subgraph "Application Node (Agent B)" - B1[Session Layer 1] - B2[Session Layer 2] - B3[Data Plane Client] - B1 --> B3 - B2 --> B3 - end - - subgraph "Control Plane" - CP[Configuration & Monitoring] - end - - A3 -.->|encrypted messages| I1 - I1 -.->|route| I2 - I2 -.->|encrypted messages| B3 - - CP -.->|manages| I1 - CP -.->|manages| I2 - - style A1 fill:#4a90e2,stroke:#2e5c8a,stroke-width:2px,color:#fff - style A2 fill:#4a90e2,stroke:#2e5c8a,stroke-width:2px,color:#fff - style A3 fill:#f39c12,stroke:#d68910,stroke-width:2px,color:#fff - style B1 fill:#4a90e2,stroke:#2e5c8a,stroke-width:2px,color:#fff - style B2 fill:#4a90e2,stroke:#2e5c8a,stroke-width:2px,color:#fff - style B3 fill:#f39c12,stroke:#d68910,stroke-width:2px,color:#fff - style I1 fill:#f39c12,stroke:#d68910,stroke-width:2px,color:#fff - style I2 fill:#f39c12,stroke:#d68910,stroke-width:2px,color:#fff - style CP fill:#7f8c8d,stroke:#5a6970,stroke-width:2px,color:#fff -``` - -### Component Distribution - -SLIM's architecture enables efficient distribution of components: - -### Pure Data Plane - -The `slim` binary and Docker images are distributed as pure -data-plane artifacts. Since SLIM routing nodes only forward messages and don't -participate in application sessions, they don't need the session layer. This -keeps the infrastructure lightweight, fast, and simple to deploy. - -### Language Bindings - -Libraries (Python, Go, etc.) include both the data plane -client and the session layer on top. Applications use these bindings to get -the full stack: secure, reliable, encrypted communication with automatic session -management. - -### Separation of Concerns - -You can run a global network of SLIM routing nodes -without any application logic, while your agents use the rich, full-featured -language bindings for their communication needs. The control plane manages the -routing infrastructure independently, ensuring the network operates efficiently. - -To get started with SLIM, see the [Getting Started with -SLIM](../slim/slim-howto.md) guide. diff --git a/docs/slim/slim-a2a.md b/docs/slim/slim-a2a.md deleted file mode 100644 index 2d5a332..0000000 --- a/docs/slim/slim-a2a.md +++ /dev/null @@ -1,296 +0,0 @@ -# SLIM A2A - -SLIM A2A is a native integration of A2A built on top of SLIM. It utilizes SLIMRPC (SLIM Remote Procedure Call) and the SLIMRPC compiler to compile A2A protobuf file and generate the necessary code to enable A2A functionality on SLIM. - -## What are SLIMRPC and the SLIMRPC Compiler? - -SLIMRPC is a framework that enables Protocol Buffers (protobuf) Remote Procedure Calls (RPC) over SLIM. This is similar to gRPC, which uses HTTP/2 as its transport layer for protobuf-based RPC. For more information, see the [SLIMRPC documentation](./slim-rpc.md). - -To compile a protobuf file and generate the clients and service stub you can use the [SLIMRPC compiler](./slim-slimrpc-compiler.md). This works in a similar way to the protoc compiler. - -For SLIM A2A we compiled the [a2a.proto](https://github.com/a2aproject/A2A/blob/v0.3.0/specification/grpc/a2a.proto) file using the SLIMRPC compiler. The generated code is in [a2a_pb2_slimrpc.py](https://github.com/agntcy/slim-a2a-python/blob/main/slima2a/types/a2a_pb2_slimrpc.py). - -## How to use SLIMA2A - -Using SLIM A2A is very similar to using the standard A2A implementation. As a reference example here we use the [echo agent](https://github.com/agntcy/slim-a2a-python/tree/main/examples/echo_agent) available in the SLIM A2A Python repository. In the following sections, we highlight and explain the key differences between the standard and SLIM A2A implementations. - -### Server Implementation - -SLIM A2A provides two approaches for setting up an A2A server: a quick start method using helper functions, and an advanced method for more control. - -#### Quick Start - -The quick start approach uses the `setup_slim_client` helper function to simplify SLIM initialization: - -```python -import asyncio -import slim_bindings -from a2a.server.request_handlers import DefaultRequestHandler -from a2a.server.tasks import InMemoryTaskStore -from slima2a import setup_slim_client -from slima2a.handler import SRPCHandler -from slima2a.types.a2a_pb2_slimrpc import add_A2AServiceServicer_to_server - -# Initialize and connect to SLIM (simplified helper) -service, local_app, local_name, conn_id = await setup_slim_client( - namespace="agntcy", - group="demo", - name="echo_agent", -) - -# Create request handler -agent_executor = MyAgentExecutor() -task_store = InMemoryTaskStore() -request_handler = DefaultRequestHandler( - agent_executor=agent_executor, - task_store=task_store, -) - -# Create servicer -servicer = SRPCHandler(agent_card, request_handler) - -# Create server -server = slim_bindings.Server.new_with_connection(local_app, local_name, conn_id) - -add_A2AServiceServicer_to_server(servicer, server) - -# Run server -await server.serve_async() -``` - -#### Advanced Setup - -??? note "Advanced Setup - Click to expand" - - For more control over the SLIM configuration, you can manually initialize all components: - - ```python - import asyncio - import slim_bindings - from a2a.server.request_handlers import DefaultRequestHandler - from a2a.server.tasks import InMemoryTaskStore - from slima2a.handler import SRPCHandler - from slima2a.types.a2a_pb2_slimrpc import add_A2AServiceServicer_to_server - - # Set the event loop for slim_bindings - slim_bindings.slim_bindings.uniffi_set_event_loop(asyncio.get_running_loop()) - - # Initialize slim_bindings service - tracing_config = slim_bindings.new_tracing_config() - runtime_config = slim_bindings.new_runtime_config() - service_config = slim_bindings.new_service_config() - tracing_config.log_level = "info" - - slim_bindings.initialize_with_configs( - tracing_config=tracing_config, - runtime_config=runtime_config, - service_config=[service_config], - ) - - service = slim_bindings.get_global_service() - - # Create local name - local_name = slim_bindings.Name("agntcy", "demo", "echo_agent") - - # Connect to SLIM - client_config = slim_bindings.new_insecure_client_config("http://localhost:46357") - conn_id = await service.connect_async(client_config) - - # Create app with shared secret - local_app = service.create_app_with_secret( - local_name, "secretsecretsecretsecretsecretsecret" - ) - - # Subscribe to local name - await local_app.subscribe_async(local_name, conn_id) - - # Create request handler - agent_executor = MyAgentExecutor() - task_store = InMemoryTaskStore() - request_handler = DefaultRequestHandler( - agent_executor=agent_executor, - task_store=task_store, - ) - - # Create servicer - servicer = SRPCHandler(agent_card, request_handler) - - # Create server - server = slim_bindings.Server.new_with_connection(local_app, local_name, conn_id) - - add_A2AServiceServicer_to_server(servicer, server) - - # Run server - await server.serve_async() - ``` - - Key differences from the standard A2A implementation: - - * **SLIM Bindings Initialization**: Set up the event loop and initialize `slim_bindings` with tracing and runtime configurations. - * **SLIM Name Creation**: Create a `slim_bindings.Name` object with namespace, group, and name components. - * **Connection Setup**: Connect to the SLIM server and create an app with a shared secret for MLS (Message Layer Security). - * **Server Creation**: Use `slim_bindings.Server.new_with_connection()` instead of a standard HTTP server. - * **Service Registration**: Register the servicer using `add_A2AServiceServicer_to_server()` from the generated SLIMRPC code. - -### Client Implementation - -Similar to the server, SLIM A2A provides both quick start and advanced client setup options. - -#### Quick Start - -```python -import asyncio -import httpx -from a2a.client import ClientFactory, minimal_agent_card -from a2a.types import Message, Part, Role, TextPart -from slima2a import setup_slim_client -from slima2a.client_transport import ClientConfig, SRPCTransport, slimrpc_channel_factory - -# Initialize and connect to SLIM (simplified helper) -service, slim_local_app, local_name, conn_id = await setup_slim_client( - namespace="agntcy", - group="demo", - name="client", -) - -# Create client config -httpx_client = httpx.AsyncClient() -client_config = ClientConfig( - supported_transports=["JSONRPC", "slimrpc"], - streaming=True, - httpx_client=httpx_client, - slimrpc_channel_factory=slimrpc_channel_factory(slim_local_app, conn_id), -) - -# Create client factory and register transport -client_factory = ClientFactory(client_config) -client_factory.register("slimrpc", SRPCTransport.create) - -# Create client with minimal agent card -agent_card = minimal_agent_card("agntcy/demo/echo_agent", ["slimrpc"]) -client = client_factory.create(card=agent_card) - -# Send message -request = Message( - role=Role.user, - message_id="request-id", - parts=[Part(root=TextPart(text="Hello, world!"))], -) - -async for event in client.send_message(request=request): - if isinstance(event, Message): - for part in event.parts: - if isinstance(part.root, TextPart): - print(part.root.text) -``` - -#### Advanced Setup - -??? note "Advanced Setup - Click to expand" - - For more control over the SLIM configuration, you can manually initialize all components: - - ```python - import asyncio - import httpx - import slim_bindings - from a2a.client import ClientFactory, minimal_agent_card - from a2a.types import Message, Part, Role, TextPart - from slima2a.client_transport import ClientConfig, SRPCTransport, slimrpc_channel_factory - - # Set the event loop for slim_bindings - slim_bindings.slim_bindings.uniffi_set_event_loop(asyncio.get_running_loop()) - - # Initialize slim_bindings service - tracing_config = slim_bindings.new_tracing_config() - runtime_config = slim_bindings.new_runtime_config() - service_config = slim_bindings.new_service_config() - tracing_config.log_level = "info" - - slim_bindings.initialize_with_configs( - tracing_config=tracing_config, - runtime_config=runtime_config, - service_config=[service_config], - ) - - service = slim_bindings.get_global_service() - - # Create local name - local_name = slim_bindings.Name("agntcy", "demo", "client") - - # Connect to SLIM - client_config_slim = slim_bindings.new_insecure_client_config("http://localhost:46357") - conn_id = await service.connect_async(client_config_slim) - - # Create app with shared secret - slim_local_app = service.create_app_with_secret( - local_name, "secretsecretsecretsecretsecretsecret" - ) - - # Subscribe to local name - await slim_local_app.subscribe_async(local_name, conn_id) - - # Create client config - httpx_client = httpx.AsyncClient() - client_config = ClientConfig( - supported_transports=["JSONRPC", "slimrpc"], - streaming=True, - httpx_client=httpx_client, - slimrpc_channel_factory=slimrpc_channel_factory(slim_local_app, conn_id), - ) - - # Create client factory and register transport - client_factory = ClientFactory(client_config) - client_factory.register("slimrpc", SRPCTransport.create) - - # Create client with minimal agent card - agent_card = minimal_agent_card("agntcy/demo/echo_agent", ["slimrpc"]) - client = client_factory.create(card=agent_card) - - # Send message - request = Message( - role=Role.user, - message_id="request-id", - parts=[Part(root=TextPart(text="Hello, world!"))], - ) - - async for event in client.send_message(request=request): - if isinstance(event, Message): - for part in event.parts: - if isinstance(part.root, TextPart): - print(part.root.text) - ``` - - Key differences from the standard A2A client: - - * **Channel Factory**: Use `slimrpc_channel_factory` instead of manually creating channels, which handles the SLIM connection details. - * **Transport Registration**: Register the `SRPCTransport` as the "slimrpc" transport in the client factory. - * **Agent Card Configuration**: Specify "slimrpc" as the transport protocol in the agent card. - * **Supported Transports**: Add "slimrpc" to the list of supported transports alongside "JSONRPC". - -### Helper Functions - -The `slima2a` package provides convenient helper functions to simplify SLIM setup: - -- `setup_slim_client(namespace, group, name, slim_url="http://localhost:46357", secret="...", log_level="info")`: Complete SLIM client setup in one call. Returns `(service, local_app, local_name, conn_id)`. -- `initialize_slim_service(log_level="info")`: Initialize SLIM service with default configuration. -- `connect_and_subscribe(service, local_name, slim_url="http://localhost:46357", secret="...")`: Connect to SLIM server and subscribe to a local name. -- `slimrpc_channel_factory(local_app, conn_id)`: Creates a channel factory function for use with A2A client configuration. - -## Configuration Parameters - -### SLIM Connection - -- `namespace`: The namespace component of the SLIM name (e.g., `agntcy`). -- `group`: The group component of the SLIM name (e.g., `demo`). -- `name`: The name component of the SLIM name (e.g., `echo_agent`). -- `slim_url`: The endpoint URL for the SLIM server (default: `http://localhost:46357`). -- `shared_secret`: A secret string used for MLS (Message Layer Security). Must be at least 32 characters. - -### Logging - -Set the log level through `tracing_config.log_level` with values like `info`, `debug`, `warn`, or `error`. - -## Running the Example - -See the [echo agent example](https://github.com/agntcy/slim-a2a-python/tree/main/examples/echo_agent) in the SLIM A2A Python repository for a complete working implementation. diff --git a/docs/slim/slim-authentication.md b/docs/slim/slim-authentication.md deleted file mode 100644 index bd83957..0000000 --- a/docs/slim/slim-authentication.md +++ /dev/null @@ -1,289 +0,0 @@ -# Identity Management - -## Overview - -Each SLIM client needs to have a valid identity. In the [SLIM Group -Communication Tutorial](slim-group-tutorial.md), we used a simple shared secret -to quickly set up identities for the clients. In a real-world scenario, you -would typically use a more secure method, such as tokens or certificates, to -authenticate clients and establish their identities. - -SLIM supports JWT (JSON Web Tokens) for identity management. Tokens can come -from an external identity provider or can be generated by the SLIM nodes -directly if you provide the necessary private key for signing the tokens and -public key for verification. Check the [Identity -Test](https://github.com/agntcy/slim/blob/slim-v1.1.0/data-plane/bindings/python/tests/test_identity.py) -for an example of how to use JWT tokens with SLIM if you have your own keys. - -If you are running your SLIM clients in a Kubernetes environment, using -[SPIRE](https://spiffe.io/) is a very common approach to give an identity -to each client. SPIRE provides a way to issue -[SPIFFE IDs](https://spiffe.io/docs/latest/spiffe-about/spiffe-concepts/#spiffe-id) to -workloads, in the form of JWT tokens, which SLIM can then use to authenticate -clients. This allows for secure and scalable identity management in distributed -systems. - -## Example: Using SPIRE with SLIM in Kubernetes (SPIRE / JWT) - -SLIM integrates well with SPIRE, as it allows you to use the JWT tokens -generated by SPIRE as client identities, and at the same time it can verify -these tokens using the key bundle provided by SPIRE. - -This section shows how to use SPIRE with SLIM to manage client identities. The following topics are covered: - -* Creating a local KIND cluster (with an in-cluster image registry). -* Installing SPIRE (server and agents). -* Building and pushing SLIM images to the local registry. -* Deploying the SLIM node (control and rendezvous components). -* Deploying two distinct SLIM client workloads, each with its own ServiceAccount (and thus its own SPIFFE ID). -* Running the point-to-point example using JWT-based authentication derived from SPIRE. - -If you already have a Kubernetes cluster or an existing SPIRE deployment, you -can adapt only the relevant subsections. - -This tutorial is based on the [SLIM examples](https://github.com/agntcy/slim/blob/slim-v1.1.0/data-plane/bindings/python/examples). - -### Prerequisites - -The following prerequisites are required: - -* [Docker](https://docs.docker.com/get-docker/) -* [kubectl](https://kubernetes.io/docs/tasks/tools/) -* [Helm](https://helm.sh/docs/intro/install/) -* [KIND](https://kind.sigs.k8s.io/docs/user/quick-start/) - -### Creating a KIND Cluster with a Local Image Registry - -The helper script below provisions a KIND cluster and configures a local -registry (localhost:5001) that the cluster’s container runtime can pull from: - -```bash -curl -L https://kind.sigs.k8s.io/examples/kind-with-registry.sh | sh -``` - -!!! note "Local Registry" - While this tutorial doesn't use the local registry, it's available at `localhost:5001` if you need to test with custom or unpublished container images. - -### Installing SPIRE - -To install SPIRE, you need to install the server, CRDs, and agents: - -```bash -helm upgrade --install \ - -n spire-server \ - spire-crds spire-crds \ - --repo https://spiffe.github.io/helm-charts-hardened/ \ - --create-namespace - -helm upgrade --install \ - -n spire-server \ - spire spire --repo \ - https://spiffe.github.io/helm-charts-hardened/ -``` - -Wait for the SPIRE components to become ready: - -```bash -kubectl get pods -n spire-server -``` - -All pods should reach Running/READY status before proceeding. - -### SPIFFE ID Strategy - -The default SPIRE server Helm chart installs a Cluster SPIFFE ID controller -object (`spire-server-spire-default`) that issues workload identities following -the pattern: - -`spiffe://domain.test/ns//sa/` - -We rely on this object by default. If you need more granular issuance (specific label -selectors, different trust domain, etc.), consult the [ClusterSPIFFEID -documentation](https://github.com/spiffe/spire-controller-manager/blob/main/docs/clusterspiffeid-crd.md). - -### Deploying the SLIM Node - -```bash -helm install \ - --create-namespace \ - -n slim \ - slim oci://ghcr.io/agntcy/slim/helm/slim:v1.1.0 -``` - -Confirm the pod is running: - -```bash -kubectl get pods -n slim -``` - -### Deploying Two Distinct Clients (separate ServiceAccounts = separate SPIFFE IDs) - -Each Deployment: - -* Has its own ServiceAccount (`slim-client-a`, `slim-client-b`). -* Mounts the SPIRE agent socket from the host (in KIND, agent runs as a - DaemonSet). -* Runs a placeholder `slim-client` container (sleep) you can exec into. - -```bash -kubectl apply -f - < connection_config.json < -``` - -**Add route directly to a node:** - -```bash -slimctl node route add org/namespace/service/0 via config.json --server= -``` - -### `version` - Version Information - -Display version and build information: - -```bash -slimctl version -``` - -### Getting Help - -Get detailed help for any command: - -```bash -slimctl --help -slimctl slim --help -slimctl slim start --help -slimctl route --help -``` - -## Usage Examples - -### Example 1: Create, Delete Route using node-id - -Add route for node `slim/a` to forward messages for `org/default/alice/0` to node `slim/b`. - -```bash -# List available nodes -slimctl controller node list -2 node(s) found -Node ID: slim/b status: CONNECTED - Connection details: - - Endpoint: 127.0.0.1:46457 - MtlsRequired: false - ExternalEndpoint: test-slim.default.svc.cluster.local:46457 -Node ID: slim/a status: CONNECTED - Connection details: - - Endpoint: 127.0.0.1:46357 - MtlsRequired: false - ExternalEndpoint: test-slim.default.svc.cluster.local:46357 - -# Add route to node slim/a -slimctl controller route add org/default/alice/0 via slim/b --node-id slim/a - -# Delete an existing route -slimctl controller route del org/default/alice/0 via slim/b --node-id slim/a -``` - -### Example 2: Create, Delete Route Using `connection_config.json` - -```bash -# Create connection configuration -cat > connection_config.json < - -Commands: - version Print version information - config Manage slimctl configuration - node Commands to interact with SLIM nodes directly - controller Commands to interact with the SLIM Control Plane - slim Commands for managing a local SLIM instance - help Print this message or the help of the given subcommand(s) -``` - -### Installing slimctl - -`slimctl` is available for multiple operating systems and architectures. - -=== "Pre-built Binaries" - - Download from the [GitHub releases page](https://github.com/agntcy/slim/releases): - - 1. Download the binary for your OS and architecture - 2. Extract the archive - 3. Move `slimctl` to a directory in your `PATH` - -=== "Homebrew (macOS)" - - If you are using macOS, you can install slimctl via Homebrew: - - ```bash - brew tap agntcy/slim https://github.com/agntcy/slim.git - brew install slimctl - ``` - -=== "Building from Source" - - **Prerequisites**: Go 1.20+, Task (taskfile.dev) - - ```bash - # From repository root - cd control-plane - task control-plane:slimctl:build - - # Binary location: .dist/bin/slimctl - ``` - -### Configuring slimctl - -`slimctl` supports configuration through a configuration file, environment variables, or command-line flags. It looks for the Control Plane configuration at the following locations: - -- `$HOME/.slimctl/config.yaml` -- `./config.yaml` (current directory) -- Via `--config` flag - -An example `config.yaml`: - -```yaml -server: "127.0.0.1:46358" -timeout: "10s" -tls: - insecure: false - ca_file: "/path/to/ca.pem" - cert_file: "/path/to/client.pem" - key_file: "/path/to/client.key" -``` - -The `server` endpoint should point to a [SLIM Control](https://github.com/agntcy/slim/tree/slim-v1.1.0/control-plane/control-plane) endpoint which is a central service managing SLIM node configurations. - -### Managing SLIM Nodes Directly - -SLIM nodes can be configured to expose a Controller endpoint. slimctl can connect to this endpoint to manage the SLIM instance directly using the `node` sub-command, bypassing the central Control Plane. - -To enable this, configure the node to host a server allowing the client to connect: - -```yaml - tracing: - log_level: info - display_thread_names: true - display_thread_ids: true - - runtime: - n_cores: 0 - thread_name: "slim-data-plane" - drain_timeout: 10s - - services: - slim/1: - dataplane: - servers: [] - clients: [] - controller: - servers: - - endpoint: "0.0.0.0:46358" - tls: - insecure: true # Or specify tls cert and key - clients: [] -``` - -**List connections on a SLIM instance:** - -```bash -slimctl node connection list --server= -``` - -**List routes on a SLIM instance:** - -```bash -slimctl node route list --server= -``` - -**Add a route to the SLIM instance:** - -```bash -slimctl node route add via --server= -``` - -**Delete a route from the SLIM instance:** - -```bash -slimctl node route del via --server= -``` - -## Key Features - -- **Centralized Node Management**: Register and manage multiple SLIM nodes from a single control point. -- **Route Configuration**: Set up message routing between nodes through the Controller. -- **Bidirectional Communication**: Supports both northbound and southbound gRPC interfaces. -- **Connection Orchestration**: Manages connections and subscriptions between SLIM nodes. - -## Architecture - -The Controller implements northbound and southbound gRPC interfaces. - -The northbound interface provides management capabilities for external systems -and administrators, such as [slimctl](#slimctl). It includes: - -- **Route Management**: Create, list, and manage message routes between nodes. -- **Connection Management**: Set up and monitor connections between SLIM nodes. -- **Node Discovery**: List registered nodes and their status. - -The southbound interface allows SLIM nodes to register with the Controller and -receive configuration updates. It includes: - -- **Node Registration**: Nodes can register themselves with the Controller. -- **Node De-registration**: Nodes can unregister when shutting down. -- **Configuration Distribution**: The Controller can push configuration updates to registered nodes. -- **Bidirectional Communication**: Supports real-time communication between the Controller and nodes. - -### Control Plane Architecture - -```mermaid -graph TB - %% User and CLI - User[👤 User/Administrator] - CLI[slimctl CLI Tool] - - %% Control Plane Components - subgraph "Control Plane" - Controller[SLIM Controller
- Northbound API
- Southbound API
- Node Registry] - Config[Configuration
Store] - end - - %% Data Plane Nodes - subgraph "Data Plane" - Node1[SLIM Node 1
- Message Routing
- Client Connections] - Node2[SLIM Node 2
- Message Routing
- Client Connections] - Node3[SLIM Node 3
- Message Routing
- Client Connections] - end - - %% Client Applications - subgraph "Applications" - App1[Client App 1] - App2[Client App 2] - App3[Client App 3] - end - - %% User interactions - User -->|Commands| CLI - CLI -->|gRPC Northbound
Port 50051| Controller - - %% Control plane interactions - Controller <-->|Store/Retrieve
Configuration| Config - - %% Southbound connections - Controller <-->|gRPC Southbound
Port 50052
Registration & Config| Node1 - Controller <-->|gRPC Southbound
Port 50052
Registration & Config| Node2 - Controller <-->|gRPC Southbound
Port 50052
Registration & Config| Node3 - - %% Inter-node communication - Node1 <-->|Message Routing| Node2 - Node2 <-->|Message Routing| Node3 - Node1 <-->|Message Routing| Node3 - - %% Application connections - App1 -->|SLIM Protocol| Node1 - App2 -->|SLIM Protocol| Node2 - App3 -->|SLIM Protocol| Node3 - - %% Styling for light/dark mode compatibility - classDef user fill:#4A90E2,stroke:#2E5D8A,stroke-width:2px,color:#FFFFFF - classDef control fill:#9B59B6,stroke:#6A3A7C,stroke-width:2px,color:#FFFFFF - classDef data fill:#27AE60,stroke:#1E8449,stroke-width:2px,color:#FFFFFF - classDef app fill:#F39C12,stroke:#D68910,stroke-width:2px,color:#FFFFFF - - class User,CLI user - class Controller,Config control - class Node1,Node2,Node3 data - class App1,App2,App3 app -``` - -### Control Flow Sequence - -```mermaid -sequenceDiagram - participant User - participant CLI as slimctl CLI - participant Controller as SLIM Controller - participant Node as SLIM Node - participant App as Client App - - %% Node Registration - Note over Node,Controller: Node Startup & Registration - Node->>Controller: Register Node (Southbound) - Controller->>Node: Registration Ack - Controller->>Controller: Store Node Info - - %% Route Management via CLI - Note over User,Controller: Route Management - User->>CLI: slimctl route add org/ns/agent via config.json - CLI->>Controller: CreateRoute Request (Northbound) - Controller->>Controller: Validate & Store Route - Controller->>Node: Push Route Configuration (Southbound) - Node->>Controller: Configuration Ack - Controller->>CLI: Route Created Response - CLI->>User: Success Message - - %% Connection Management - Note over User,Controller: Connection Management - User->>CLI: slimctl connection list --node-id=slim/1 - CLI->>Controller: ListConnections Request (Northbound) - Controller->>Controller: Retrieve Connection Info - Controller->>CLI: Connections List Response - CLI->>User: Display Connections - - %% Application Communication - Note over App,Node: Application Messaging - App->>Node: Connect & Subscribe - Node->>App: Connection Established - App->>Node: Publish Message - Node->>Node: Route Message via Controller Config - - %% Node Status Updates - Note over Node,Controller: Status Monitoring - Node->>Controller: Status Update (Southbound) - Controller->>Controller: Update Node Status -``` - -## Usage - -## Prerequisites - -Go 1.24 or later is required for running the SLIM Controller. - -Task runner is recommended for Taskfile commands. - -### Building the Controller - -The Controller can be built by running the following task: - -```task -# Build all Controller components -task control-plane:build - -# Or build just the Controller binary -task control-plane:control-plane:build -``` - -### Starting the Controller - -The Controller can be started by running the following task: - -```bash -# Start the Controller service -task control-plane:control-plane:run -``` - -Alternatively, start the Controller with the Docker image: - -```bash -docker run ghcr.io/agntcy/slim/control-plane:0.0.1 -``` - -Or use the following to also add a configuration file: - -```bash -docker run -v ./config.yaml:/config.yaml ghcr.io/agntcy/slim/control-plane:0.0.1 -c /config.yaml -``` - -### Managing Nodes - -Nodes can register themselves with the Controller upon startup. Once registered, the controller can communicate with nodes using the same connection. - -To enable self-registration, configure the nodes with the Controller address: - -```yaml - tracing: - log_level: info - display_thread_names: true - display_thread_ids: true - - runtime: - n_cores: 0 - thread_name: "slim-data-plane" - drain_timeout: 10s - - services: - slim/1: - dataplane: - servers: [] - clients: [] - controller: - servers: [] - clients: - - endpoint: "http://:50052" - tls: - insecure: true -``` - -Routes between SLIM nodes are automatically created by Controller upon receiving new subscriptions from clients. - -Nodes can be managed through slimctl. Although routes are automatically created for client subscription you can still add/remove routes manually. - -For more information, see the [slimctl](#slimctl). diff --git a/docs/slim/slim-data-plane-config.md b/docs/slim/slim-data-plane-config.md deleted file mode 100644 index ce47466..0000000 --- a/docs/slim/slim-data-plane-config.md +++ /dev/null @@ -1,1586 +0,0 @@ -# SLIM Data Plane Configuration - -This document provides comprehensive documentation for configuring the SLIM data plane. The configuration is written in YAML format and defines how the data plane runtime, services, authentication, and observability components operate. - -This documentation corresponds to the JSON schemas in the SLIM repository: - -- [Client Configuration Schema](https://github.com/agntcy/slim/blob/slim-v1.1.0/data-plane/core/config/src/grpc/schema/client-config.schema.json) -- [Server Configuration Schema](https://github.com/agntcy/slim/blob/slim-v1.1.0/data-plane/core/config/src/grpc/schema/server-config.schema.json) - -## Configuration Structure Overview - -The SLIM configuration file consists of three main sections: - -=== "Tracing" - Observability and logging configuration - ```yaml - tracing: - log_level: info - opentelemetry: - enabled: true - ``` - -=== "Runtime" - Runtime behavior configuration - ```yaml - runtime: - n_cores: 0 - drain_timeout: "10s" - ``` - -=== "Services" - Services configuration - ```yaml - services: - slim/0: - dataplane: - servers: [...] - clients: [...] - ``` - -## Top-Level Configuration Sections - -### Tracing Configuration - -The `tracing` section configures logging, observability, and OpenTelemetry integration. - -#### Basic Tracing Options - -```yaml -tracing: - # Logging level - controls verbosity of log output - # Available options: trace, debug, info, warn, error - # Default: info - log_level: debug - - # Whether to display thread names in log output - # Default: true - display_thread_names: true - - # Whether to display thread IDs in log output - # Default: false - display_thread_ids: true - - # Additional log filtering (optional) - # Can be used to filter logs by module or target - # Default: "info" - filter: "slim=debug" -``` - -#### OpenTelemetry Configuration - -```yaml -tracing: - opentelemetry: - # Enable OpenTelemetry integration for distributed tracing - # Default: false - enabled: true - - # Service name for telemetry identification - # Default: "slim-data-plane" - service_name: "slim-data-plane" - - # Service version for telemetry - # Default: "v0.1.0" - service_version: "v0.2.0" - - # Environment identifier (e.g., prod, staging, dev) - # Default: "development" - environment: "production" - - # Metrics collection interval in seconds - # Default: 30 - metrics_interval_secs: 60 - - # gRPC configuration for OpenTelemetry exporter - grpc: - endpoint: "http://otel-collector:4317" - tls: - insecure: true -``` - -### Runtime Configuration - -The `runtime` section configures the async runtime behavior and resource allocation. - -```yaml -runtime: - # Number of worker threads for the async runtime - # 0 = use all available CPU cores - # Default: 0 - n_cores: 4 - - # Thread name prefix for runtime worker threads - # Default: "slim" - thread_name: "slim-data-plane" - - # Timeout for graceful shutdown - how long to wait for tasks to complete - # Format: "s", "ms", "m", "h" - # Default: "10s" - drain_timeout: "30s" -``` - -### Services Configuration - -The `services` section defines SLIM service instances and their network configurations. Each service is identified by a unique ID in the format `slim/`. - -## Service Configuration - -### Basic Service Structure - -```yaml -services: - # Service identifier - format: slim/ - slim/0: - # Optional node ID override - # If not specified, uses the service identifier - node_id: "my-node-01" - - # Data plane API configuration - dataplane: - servers: [...] # Server endpoints this instance will listen on - clients: [...] # Client connections this instance will make - - # Control plane API configuration (optional) - controller: - servers: [...] # Control plane server endpoints - clients: [...] # Control plane client connections -``` - -## TLS Configuration - -TLS configuration is used throughout SLIM for securing connections. The same TLS configuration structure applies to: - -- Servers (`dataplane.servers[].tls`, `controller.servers[].tls`) -- Clients (`dataplane.clients[].tls`, `controller.clients[].tls`) -- Proxies (`dataplane.clients[].proxy.tls`) - - -All TLS options documented in this section can be used in any context that accepts a `tls` configuration block. The behavior adapts based on the context (server vs client). - -### TLS Modes - -=== "Insecure Mode (Development)" - ```yaml - tls: - insecure: true # Disable TLS (not recommended for production) - ``` - -=== "Secure Mode (Production)" - ```yaml - tls: - insecure: false # Default: false - requires certificates - source: - type: file - cert: "./certs/cert.pem" - key: "./certs/key.pem" - ``` - -!!! warning "TLS Configuration Required" - When `insecure: false` (the default), you must provide certificates via `source`. The service does not start without them. - -### Certificate Sources (`source`) - -The `source` field provides the certificate and private key for the TLS endpoint. - -Certificate source usage varies by component type: - -- Servers provide the server's identity certificate -- Clients provide the client certificate for mutual TLS (mTLS) - -=== "File-based Certificates" - ```yaml - tls: - source: - type: file - cert: "./certs/cert.pem" - key: "./certs/key.pem" - ``` - -=== "Inline PEM Certificates" - ```yaml - tls: - source: - type: pem - cert: | - -----BEGIN CERTIFICATE----- - ... - -----END CERTIFICATE----- - key: | - -----BEGIN PRIVATE KEY----- - ... - -----END PRIVATE KEY----- - ``` - -=== "SPIRE Integration" - ```yaml - tls: - source: - type: spire - socket_path: "/run/spire/sockets/agent.sock" - jwt_audiences: ["slim", "dataplane"] - target_spiffe_id: "spiffe://example.org/service" - trust_domains: ["example.org"] - ``` - -=== "No Certificate" - ```yaml - tls: - source: - type: none # No certificate configured - ``` - -### CA Certificate Sources - -CA sources are used for certificate verification. The field name differs based on context: - -- Servers use `client_ca` to verify client certificates (for mTLS) -- Clients use `ca_source` to verify server certificates -- Proxies use `ca_source` to verify proxy server certificates - -#### Server Client CA (`client_ca`) - -Used by servers to verify client certificates when mTLS is required. - -=== "File-based CA" - ```yaml - tls: - client_ca: - type: file - path: "./certs/ca-cert.pem" - ``` - -=== "Inline PEM CA" - ```yaml - tls: - client_ca: - type: pem - data: | - -----BEGIN CERTIFICATE----- - ... - -----END CERTIFICATE----- - ``` - -=== "SPIRE Bundle" - ```yaml - tls: - client_ca: - type: spire - socket_path: "/run/spire/sockets/agent.sock" - trust_domains: ["example.org"] - ``` - -=== "No Client Verification" - ```yaml - tls: - client_ca: - type: none # No client certificate verification - ``` - -#### Client CA Source (`ca_source`) - -Used by clients to verify server certificates. - -=== "File-based CA" - ```yaml - tls: - ca_source: - type: file - path: "./certs/ca-cert.pem" - ``` - -=== "Inline PEM CA" - ```yaml - tls: - ca_source: - type: pem - data: | - -----BEGIN CERTIFICATE----- - ... - -----END CERTIFICATE----- - ``` - -=== "SPIRE Bundle" - ```yaml - tls: - ca_source: - type: spire - socket_path: "/run/spire/sockets/agent.sock" - trust_domains: ["example.org"] - ``` - -=== "No CA" - ```yaml - tls: - ca_source: - type: none # No CA configured - ``` - -### TLS Options - -```yaml -tls: - # TLS version constraint - # Options: "tls1.2", "tls1.3" - # Default: "tls1.3" - tls_version: "tls1.3" - - # Include system CA certificates (CLIENT ONLY) - # Default: true - # Only used by clients when verifying servers - include_system_ca_certs_pool: true - - # Skip server name verification (CLIENT ONLY - INSECURE) - # Default: false - insecure_skip_verify: false - - # Reload client CA file when modified (SERVER ONLY - NOT YET IMPLEMENTED) - # Default: false - # reload_client_ca_file: false -``` - -The following options are context-specific: - -- `include_system_ca_certs_pool` - Only used by clients, ignored by servers -- `insecure_skip_verify` - Only used by clients, ignored by servers -- `client_ca` - Only used by servers, not available for clients -- `ca_source` - Used by clients and proxies, exists in server schema but unused - -### TLS Examples by Context - -=== "Server with mTLS" - ```yaml - dataplane: - servers: - - endpoint: "0.0.0.0:46357" - tls: - insecure: false - # Server's certificate - source: - type: file - cert: "./certs/server-cert.pem" - key: "./certs/server-key.pem" - # Verify client certificates - client_ca: - type: file - path: "./certs/ca-cert.pem" - tls_version: "tls1.3" - ``` - -=== "Client with mTLS" - ```yaml - dataplane: - clients: - - endpoint: "remote-slim:46357" - tls: - insecure: false - # Client's certificate for mTLS - source: - type: file - cert: "./certs/client-cert.pem" - key: "./certs/client-key.pem" - # Verify server certificate - ca_source: - type: file - path: "./certs/ca-cert.pem" - include_system_ca_certs_pool: true - tls_version: "tls1.3" - ``` - -=== "HTTPS Proxy" - ```yaml - dataplane: - clients: - - endpoint: "remote-slim:46357" - proxy: - url: "https://proxy.example.com:8443" - tls: - insecure: false - # Verify proxy server - ca_source: - type: file - path: "./certs/proxy-ca.crt" - ``` - -## Authentication Configuration - -Authentication configuration is used throughout SLIM for securing API access. The same authentication structure applies to: - -- Servers (`dataplane.servers[].auth`, `controller.servers[].auth`) -- Clients (`dataplane.clients[].auth`, `controller.clients[].auth`) - -All authentication options documented in this section can be used in any context that accepts an `auth` configuration block. The behavior adapts based on the context (server vs client). - -### Authentication Types - -=== "Basic Authentication" - Username and password authentication. - - **Server (Verification):** - ```yaml - auth: - type: basic - username: "admin" - password: "secret123" - # Or use environment variables: - # password: "${env:ADMIN_PASSWORD}" - ``` - - **Client (Credentials):** - ```yaml - auth: - type: basic - username: "client-user" - password: "${env:CLIENT_PASSWORD}" - ``` - -=== "JWT - Dynamic Generation" - Generate JWT tokens on-the-fly with signing. - - **Server (Verification):** - ```yaml - auth: - type: jwt - claims: - audience: ["slim-dataplane"] - issuer: "slim-auth-service" - subject: "dataplane-access" - custom_claims: - role: "dataplane-user" - permissions: "read,write" - duration: "1h" - key: - type: decoding # Verify tokens - algorithm: "ES256" - format: pem - key: - file: "./keys/jwt-public.pem" - ``` - - **Client (Signing):** - ```yaml - auth: - type: jwt - claims: - audience: ["remote-slim"] - issuer: "local-slim" - subject: "client-connection" - custom_claims: - client_id: "slim-instance-01" - duration: "1h" - key: - type: encoding # Sign tokens - algorithm: "ES256" - format: pem - key: - file: "./keys/jwt-private.pem" - ``` - -=== "JWT - Shared Secret (HMAC)" - Use shared secrets for JWT signing and verification. - - **Server (Verification):** - ```yaml - auth: - type: jwt - claims: - audience: ["slim-dataplane"] - issuer: "slim-client" - key: - type: decoding - algorithm: "HS256" - format: pem - key: - data: "my-secure-shared-secret" - # Or: file: "/run/secrets/jwt-shared-secret" - ``` - - **Client (Signing):** - ```yaml - auth: - type: jwt - claims: - audience: ["slim-dataplane"] - issuer: "slim-client" - duration: "1h" - key: - type: encoding - algorithm: "HS256" # Must match server - format: pem - key: - data: "my-secure-shared-secret" # Must match server - # Or: file: "/run/secrets/jwt-shared-secret" - ``` - -=== "Static JWT Token (Client Only)" - Use pre-generated JWT tokens from files. - - ```yaml - auth: - type: static_jwt - file: "/run/secrets/jwt-token" - duration: "1h" # Cache validity before re-reading file - ``` - - !!! note - Static JWT authentication is only available for clients. - -=== "JWT - Autoresolve" - Automatically determine encoding/decoding based on context. - - ```yaml - auth: - type: jwt - claims: - audience: ["remote-slim"] - duration: "1h" - key: - type: autoresolve # Auto-detect encoding vs decoding - ``` - -=== "No Authentication" - ```yaml - auth: - type: none - ``` - -### JWT Key Configuration - -JWT keys support multiple formats and algorithms. - -#### Key Types - -The following key types are supported: - -- `encoding` for signing JWTs (typically client-side) -- `decoding` for verifying JWTs (typically server-side) -- `autoresolve` automatically determine based on context - -#### Key Formats - -=== "PEM Format" - ```yaml - key: - type: encoding - algorithm: "RS256" - format: pem - key: - file: "./keys/private.pem" - # OR inline: - # data: | - # -----BEGIN PRIVATE KEY----- - # ... - # -----END PRIVATE KEY----- - ``` - -=== "JWK Format" - ```yaml - key: - type: decoding - algorithm: "RS256" - format: jwk - key: - file: "./keys/public.jwk" - # OR inline: - # data: '{"kty":"RSA","n":"...","e":"AQAB"}' - ``` - -=== "JWKS Format" - ```yaml - key: - type: decoding - algorithm: "RS256" - format: jwks - key: - file: "./keys/jwks.json" - # OR inline: - # data: '{"keys":[{"kty":"RSA","n":"...","e":"AQAB"}]}' - ``` - -!!! tip "Shared Secret Security" - - Both client and server must use the **same** shared secret - - Both must use the **same** HMAC algorithm (HS256, HS384, or HS512) - - Consider using environment variable substitution: `data: "${env:JWT_SECRET}"` - - For production, store secrets in secure secret management systems - -#### SPIRE Authentication - -SPIRE does not have a separate `auth` type. Instead, SPIRE provides authentication through the following mechanisms: - -- TLS mutual authentication using `tls.source: { type: spire }` for certificate-based authentication -- JWT SVIDs using `auth: { type: jwt }` or `auth: { type: static_jwt }` with SPIRE-issued JWT tokens - -See the [Native SPIRE Integration](#native-spire-integration) section for complete examples. - -## Server Configuration - -Servers define endpoints that the SLIM instance listens on for incoming connections. - -### Server Endpoint Configuration - -#### Network Address - -```yaml -dataplane: - servers: - - # REQUIRED: Listen address - endpoint: "0.0.0.0:46357" - - # TLS configuration (see TLS Configuration section) - tls: - insecure: false - source: - type: file - cert: "./certs/server-cert.pem" - key: "./certs/server-key.pem" - - # Authentication (see Authentication Configuration section) - auth: - type: none -``` - -#### Unix Socket - -```yaml -dataplane: - servers: - - # REQUIRED: Unix socket path - endpoint: "unix:///var/run/slim/dataplane.sock" - - # TLS must be set to insecure for unix sockets - tls: - insecure: true -``` - -### Server Connection Settings - -```yaml -servers: - - endpoint: "0.0.0.0:46357" - - # HTTP/2 configuration - # Default: true - http2_only: true - - # Maximum size (in MiB) of messages - # Default: 4 - max_frame_size: 4 - - # Connection limits - # Default: 100 - max_concurrent_streams: 100 - # Default: null (unlimited) - max_header_list_size: 16384 # 16 KiB - - # Buffer sizes for gRPC server - # Default: 1048576 (1 MiB) for both - read_buffer_size: 1048576 # 1 MiB - write_buffer_size: 1048576 # 1 MiB - - # Connection keepalive settings - keepalive: - max_connection_idle: "3600s" # Close idle connections after 1 hour - max_connection_age: "7200s" # Maximum connection lifetime - max_connection_age_grace: "300s" # Grace period before force close - time: "120s" # Keepalive ping interval - timeout: "20s" # Keepalive ping timeout - - # Arbitrary user-provided metadata (optional) - metadata: - role: "ingress" - replicas: 3 - environment: "production" - tags: - - "dataplane" - - "grpc" -``` - -## Client Configuration - -Clients define outbound connections that the SLIM instance establishes to other services. - -### Client Endpoint Configuration - -#### Network Address - -```yaml -dataplane: - clients: - - # REQUIRED: Target endpoint - endpoint: "http://remote-slim:46357" - - # TLS configuration (see TLS Configuration section) - tls: - insecure: false - ca_source: - type: file - path: "./certs/ca-cert.pem" - - # Authentication (see Authentication Configuration section) - auth: - type: none -``` - -#### Unix Socket - -```yaml -dataplane: - clients: - - # REQUIRED: Unix socket path - endpoint: "unix:///var/run/slim/remote-node.sock" - - # TLS must be set to insecure for unix sockets - tls: - insecure: true -``` - -### Client Connection Settings - -```yaml -clients: - - endpoint: "http://remote-slim:46357" - - # Optional TLS SNI server name override - # Default: null (uses host from endpoint/origin) - server_name: "service.example.com" - - # Optional origin for client requests - origin: "https://my-client.example.com" - - # Connection timeouts (0s means no timeout) - # Default: 0s for both - connect_timeout: "10s" - request_timeout: "30s" - - # Buffer configuration - buffer_size: 8192 - - # Custom headers - headers: - x-client-id: "slim-instance-01" - x-environment: "production" - - # Rate limiting - # Format: "/" - rate_limit: "100/60" # 100 requests per minute -``` - -### HTTP Proxy Configuration - -Proxy configuration supports both HTTP and HTTPS proxies with optional authentication. - -=== "HTTP Proxy" - ```yaml - dataplane: - clients: - - endpoint: "remote-slim:46357" - proxy: - url: "http://proxy.example.com:8080" - username: "proxy-user" - password: "${env:PROXY_PASSWORD}" - headers: - x-proxy-client: "slim-dataplane" - ``` - -=== "HTTPS Proxy" - ```yaml - dataplane: - clients: - - endpoint: "remote-slim:46357" - proxy: - url: "https://secure-proxy.example.com:8443" - username: "${env:PROXY_USER}" - password: "${env:PROXY_PASS}" - # TLS configuration for proxy connection - # (see TLS Configuration section) - tls: - insecure: false - ca_source: - type: file - path: "./certs/proxy-ca.crt" - headers: - x-department: "engineering" - ``` - -### Connection Keepalive - -```yaml -clients: - - endpoint: "remote-slim:46357" - keepalive: - tcp_keepalive: "60s" - http2_keepalive: "60s" - timeout: "10s" - keep_alive_while_idle: false -``` - -### Backoff Configuration - -=== "Exponential Backoff" - ```yaml - dataplane: - clients: - - endpoint: "remote-slim:46357" - backoff: - type: exponential - base: 100 # Base delay in milliseconds (default: 100) - factor: 2 # Multiply delay by this factor each retry (default: 1) - jitter: true # Add random variation (default: true) - max_delay: "10s" # Maximum delay between retries (default: "1s") - max_attempts: 10 # Maximum number of retry attempts (default: unlimited) - ``` - -=== "Fixed Interval Backoff" - ```yaml - dataplane: - clients: - - endpoint: "remote-slim:46357" - backoff: - type: fixed_interval - interval: "2s" # Wait 2 seconds between each retry (default: "1s") - max_attempts: 5 # Maximum number of retry attempts (default: unlimited) - ``` - -!!! info "Default Backoff" - Default: exponential with base=100ms, factor=1, jitter=true, max_delay=1s, unlimited attempts - -## Native SPIRE Integration - -SLIM has native support for SPIFFE/SPIRE Workload API for automatic certificate management and zero-trust authentication. - -### Server with SPIRE - -```yaml -dataplane: - servers: - - endpoint: "0.0.0.0:46357" - tls: - # Get server certificate from SPIRE - source: - type: spire - # Optional socket path (defaults to SPIFFE_ENDPOINT_SOCKET env var) - socket_path: "/run/spire/sockets/agent.sock" - # JWT SVID audiences - jwt_audiences: ["slim", "dataplane"] - # Optional target SPIFFE ID for JWT SVIDs - target_spiffe_id: "spiffe://example.org/dataplane" - # Optional trust domains override - trust_domains: ["example.org", "partner.org"] - - # Verify client certificates using SPIRE bundle - client_ca: - type: spire - socket_path: "/run/spire/sockets/agent.sock" - trust_domains: ["example.org"] - - auth: - type: jwt - claims: - audience: ["slim-cluster"] - key: - type: decoding - algorithm: "RS256" - format: jwks - key: - # JWT bundles from SPIRE - file: "/run/spire/jwt-bundle.json" -``` - -### Client with SPIRE - -```yaml -dataplane: - clients: - - endpoint: "remote-service:46357" - tls: - # Use SPIRE for client certificate - source: - type: spire - socket_path: "/run/spire/sockets/agent.sock" - jwt_audiences: ["slim"] - target_spiffe_id: "spiffe://example.org/remote-service" - trust_domains: ["example.org"] - - # Verify server using SPIRE bundle - ca_source: - type: spire - socket_path: "/run/spire/sockets/agent.sock" - trust_domains: ["example.org"] - - # Optional: Add SPIFFE ID to headers - headers: - x-spiffe-id: "${env:SPIFFE_ID}" -``` - -When configuring SPIRE, keep the following in mind: - -- Automatic rotation: SPIRE automatically rotates certificates. -- Socket path: If not specified, uses `SPIFFE_ENDPOINT_SOCKET` environment variable. -- Trust domains: When not specified, SLIM derives from the current SVID. -- JWT audiences: Used when requesting JWT SVIDs from SPIRE. -- Zero-trust: SPIRE provides cryptographic workload identity without long-lived secrets. - -## Reference: JWT Algorithms - -### Symmetric Algorithms (HMAC - Shared Secret) - -These algorithms use a shared secret for both signing and verification. Recommended use case is when both client and server can securely share a secret. The same secret is used for signing (client) and verification (server). - -| Algorithm | Description | Key Size | -|-----------|-------------|----------| -| `HS256` | HMAC using SHA-256 ⭐ | Any | -| `HS384` | HMAC using SHA-384 | Any | -| `HS512` | HMAC using SHA-512 | Any | - -### Asymmetric Algorithms (RSA - Public/Private Key) - -The recommended use case is when client and server have different secrets. Client signs with private key, server verifies with public key. - -| Algorithm | Description | Key Size | -|-----------|-------------|----------| -| `RS256` | RSA signature with SHA-256 ⭐ | 2048+ bits | -| `RS384` | RSA signature with SHA-384 | 2048+ bits | -| `RS512` | RSA signature with SHA-512 | 2048+ bits | - -### Asymmetric Algorithms (RSA-PSS) - -| Algorithm | Description | Key Size | -|-----------|-------------|----------| -| `PS256` | RSA-PSS signature with SHA-256 | 2048+ bits | -| `PS384` | RSA-PSS signature with SHA-384 | 2048+ bits | -| `PS512` | RSA-PSS signature with SHA-512 | 2048+ bits | - -### Asymmetric Algorithms (ECDSA - Public/Private Key) - -The recommended use case is when client and server have different secrets. Client signs with private key, server verifies with public key. - -| Algorithm | Description | Curve | -|-----------|-------------|-------| -| `ES256` | ECDSA using P-256 and SHA-256 ⭐ | P-256 | -| `ES384` | ECDSA using P-384 and SHA-384 | P-384 | - -### EdDSA - -| Algorithm | Description | -|-----------|-------------| -| `EdDSA` | EdDSA signature algorithms | - -## Configuration Value Substitution - -SLIM supports dynamic configuration value substitution from environment variables and files. - -### Environment Variable Substitution - -Configuration values can reference environment variables using the `${env:VARIABLE_NAME}` syntax: - -```yaml -tracing: - log_level: "${env:LOG_LEVEL}" - -runtime: - n_cores: "${env:WORKER_THREADS}" - -services: - slim/0: - dataplane: - servers: - - endpoint: "0.0.0.0:${env:LISTEN_PORT}" - auth: - type: basic - username: "${env:AUTH_USERNAME}" - password: "${env:AUTH_PASSWORD}" -``` - -### File Content Substitution - -Configuration values can reference file contents using the `${file:PATH}` syntax: - -```yaml -services: - slim/0: - dataplane: - servers: - - endpoint: "0.0.0.0:46357" - tls: - source: - type: file - cert: "/etc/slim/certs/server.crt" - key: "/etc/slim/certs/server.key" - auth: - type: basic - # Load password from a secure file - password: "${file:/run/secrets/admin_password}" -``` - -### Substitution Examples - -=== "Kubernetes Secrets" - ```yaml - # Perfect for Kubernetes deployments with mounted secrets - services: - slim/0: - dataplane: - servers: - - endpoint: "0.0.0.0:46357" - tls: - source: - type: file - cert: "/var/run/secrets/kubernetes.io/tls/tls.crt" - key: "/var/run/secrets/kubernetes.io/tls/tls.key" - auth: - type: jwt - key: - type: decoding - algorithm: "RS256" - format: pem - key: - file: "/var/run/secrets/jwt/public.key" - ``` - -=== "Docker Secrets" - ```yaml - # For Docker Swarm or Compose with secrets - services: - slim/0: - dataplane: - servers: - - endpoint: "0.0.0.0:46357" - auth: - type: basic - username: "${env:AUTH_USERNAME}" - # Docker secret mounted as file - password: "${file:/run/secrets/db_password}" - ``` - -=== "Environment Variables" - ```yaml - tracing: - log_level: "${env:LOG_LEVEL}" - opentelemetry: - service_name: "${env:SERVICE_NAME}" - environment: "${env:ENVIRONMENT}" - - services: - slim/0: - node_id: "${env:POD_NAME}" - dataplane: - servers: - - endpoint: "0.0.0.0:${env:DATAPLANE_PORT}" - ``` - -The following rules apply to substitution: - -* Exact Replacement: The entire value must be a substitution expression. - - Valid: `password: "${env:PASSWORD}"` - - Invalid: `password: "prefix-${env:PASSWORD}-suffix"` -* Error Handling: If substitution fails, configuration loading will fail. -* File Content: Reads entire file content as string, including newlines. -* Security: File paths are relative to working directory or absolute. - -## Duration Format - -SLIM uses two different duration formats depending on the field. - -### DurationString Format (Most Fields) - -Most duration fields use a human-readable string format: - -```yaml -# Supported units: y, w, d, h, m, s, ms -timeout: "30s" -max_age: "1h30m" -interval: "500ms" -connect_timeout: "1m30s" -``` - -**Examples:** - -- `"10s"` - 10 seconds -- `"5m"` - 5 minutes -- `"1h30m"` - 1 hour 30 minutes -- `"2d"` - 2 days -- `"100ms"` - 100 milliseconds - -## Complete Configuration Examples - -### Development Configuration - -```yaml -# config/development.yaml -tracing: - log_level: debug - display_thread_names: true - display_thread_ids: true - -runtime: - n_cores: 0 - thread_name: "slim-dev" - drain_timeout: "5s" - -services: - slim/0: - dataplane: - servers: - - endpoint: "0.0.0.0:46357" - tls: - insecure: true - clients: [] -``` - -### Production Configuration with mTLS - -```yaml -# config/production.yaml -tracing: - log_level: info - display_thread_names: false - display_thread_ids: false - opentelemetry: - enabled: true - service_name: "slim-dataplane" - service_version: "v1.0.0" - environment: "production" - -runtime: - n_cores: 0 - thread_name: "slim-prod" - drain_timeout: "30s" - -services: - slim/0: - node_id: "${env:NODE_ID}" - dataplane: - servers: - - endpoint: "0.0.0.0:46357" - tls: - insecure: false - source: - type: file - cert: "/etc/slim/certs/server.crt" - key: "/etc/slim/certs/server.key" - client_ca: - type: file - path: "/etc/slim/certs/ca.crt" - tls_version: "tls1.3" - keepalive: - max_connection_idle: "1800s" - time: "300s" - timeout: "60s" - - clients: - - endpoint: "peer1.example.com:46357" - tls: - insecure: false - ca_source: - type: file - path: "/etc/slim/certs/ca.crt" - source: - type: file - cert: "/etc/slim/certs/client.crt" - key: "/etc/slim/certs/client.key" - connect_timeout: "15s" - request_timeout: "120s" - backoff: - type: exponential - base: 100 - factor: 2 - jitter: true - max_delay: "10s" - max_attempts: 5 - - controller: - servers: - - endpoint: "0.0.0.0:46358" - tls: - insecure: false - source: - type: file - cert: "/etc/slim/certs/server.crt" - key: "/etc/slim/certs/server.key" -``` - -### JWT Authentication Configuration - -```yaml -# config/jwt-auth.yaml -tracing: - log_level: info - -runtime: - n_cores: 4 - thread_name: "slim-jwt" - drain_timeout: "15s" - -services: - slim/0: - dataplane: - servers: - - endpoint: "0.0.0.0:46357" - tls: - insecure: false - source: - type: file - cert: "./certs/server.crt" - key: "./certs/server.key" - auth: - type: jwt - claims: - audience: ["slim-cluster"] - issuer: "slim-auth" - subject: "dataplane-access" - key: - type: decoding - algorithm: "ES256" - format: pem - key: - file: "./keys/jwt-public.pem" - - clients: - - endpoint: "remote.example.com:46357" - tls: - ca_source: - type: file - path: "./certs/ca.crt" - auth: - type: jwt - claims: - audience: ["remote-slim"] - issuer: "local-slim" - duration: "2h" - key: - type: encoding - algorithm: "ES256" - format: pem - key: - file: "./keys/jwt-private.pem" -``` - -### Kubernetes Deployment Configuration - -```yaml -# config/kubernetes.yaml -tracing: - log_level: "${env:LOG_LEVEL}" - opentelemetry: - enabled: true - service_name: "${env:SERVICE_NAME}" - environment: "${env:ENVIRONMENT}" - grpc: - endpoint: "${env:OTEL_COLLECTOR_ENDPOINT}" - -runtime: - n_cores: "${env:WORKER_THREADS}" - thread_name: "${env:SERVICE_NAME}" - drain_timeout: "${env:SHUTDOWN_TIMEOUT}" - -services: - slim/0: - node_id: "${env:POD_NAME}" - dataplane: - servers: - - endpoint: "0.0.0.0:${env:DATAPLANE_PORT}" - tls: - insecure: false - source: - type: file - cert: "/var/run/secrets/kubernetes.io/tls/tls.crt" - key: "/var/run/secrets/kubernetes.io/tls/tls.key" - client_ca: - type: file - path: "/var/run/secrets/kubernetes.io/ca/ca.crt" - auth: - type: jwt - claims: - audience: ["${env:JWT_AUDIENCE}"] - issuer: "${env:JWT_ISSUER}" - key: - type: decoding - algorithm: "RS256" - format: pem - key: - file: "/var/run/secrets/jwt/public.key" - - clients: - - endpoint: "${env:PEER_ENDPOINT}" - proxy: - url: "${env:HTTP_PROXY}" - username: "${env:PROXY_USER}" - password: "${env:PROXY_PASSWORD}" - tls: - ca_source: - type: file - path: "/var/run/secrets/kubernetes.io/ca/ca.crt" - source: - type: file - cert: "/var/run/secrets/kubernetes.io/tls/tls.crt" - key: "/var/run/secrets/kubernetes.io/tls/tls.key" - headers: - x-service-account: "${file:/var/run/secrets/kubernetes.io/serviceaccount/token}" - x-cluster-id: "${env:CLUSTER_ID}" - backoff: - type: exponential - base: 100 - factor: 2 - jitter: true - max_delay: "30s" - max_attempts: 10 - - controller: - servers: - - endpoint: "0.0.0.0:${env:CONTROLLER_PORT}" - tls: - source: - type: file - cert: "/var/run/secrets/kubernetes.io/tls/tls.crt" - key: "/var/run/secrets/kubernetes.io/tls/tls.key" - auth: - type: basic - username: "${env:CONTROLLER_USER}" - password: "${file:/var/run/secrets/controller/password}" -``` - -### Native SPIRE Zero Trust Configuration - -```yaml -# config/spire-native.yaml -tracing: - log_level: "${env:LOG_LEVEL}" - opentelemetry: - enabled: true - service_name: "slim-spire-${env:SPIFFE_ID}" - environment: "${env:ENVIRONMENT}" - -runtime: - n_cores: "${env:WORKER_THREADS}" - thread_name: "slim-spire" - drain_timeout: "30s" - -services: - slim/0: - node_id: "${env:SPIFFE_ID}" - - dataplane: - servers: - - endpoint: "0.0.0.0:${env:DATAPLANE_PORT}" - tls: - insecure: false - - # Automatically rotated certificates from SPIRE - source: - type: spire - socket_path: "/run/spire/sockets/agent.sock" - jwt_audiences: ["slim", "dataplane"] - trust_domains: ["${env:SPIFFE_TRUST_DOMAIN}"] - - # Client verification using SPIRE bundle - client_ca: - type: spire - socket_path: "/run/spire/sockets/agent.sock" - trust_domains: ["${env:SPIFFE_TRUST_DOMAIN}"] - - tls_version: "tls1.3" - - auth: - type: jwt - claims: - audience: ["spiffe://${env:SPIFFE_TRUST_DOMAIN}/slim-cluster"] - issuer: "spiffe://${env:SPIFFE_TRUST_DOMAIN}/slim-issuer" - subject: "${env:SPIFFE_ID}" - custom_claims: - spiffe_id: "${env:SPIFFE_ID}" - trust_domain: "${env:SPIFFE_TRUST_DOMAIN}" - key: - type: decoding - algorithm: "RS256" - format: jwks - key: - file: "/run/spire/jwt-bundle.json" - - keepalive: - max_connection_idle: "600s" - time: "60s" - timeout: "10s" - - clients: - - endpoint: "${env:PEER_ENDPOINT}" - tls: - insecure: false - - source: - type: spire - socket_path: "/run/spire/sockets/agent.sock" - target_spiffe_id: "${env:PEER_SPIFFE_ID}" - jwt_audiences: ["slim"] - trust_domains: ["${env:SPIFFE_TRUST_DOMAIN}"] - - ca_source: - type: spire - socket_path: "/run/spire/sockets/agent.sock" - trust_domains: ["${env:SPIFFE_TRUST_DOMAIN}"] - - headers: - x-spiffe-id: "${env:SPIFFE_ID}" - x-trust-domain: "${env:SPIFFE_TRUST_DOMAIN}" - - connect_timeout: "10s" - request_timeout: "30s" - - backoff: - type: exponential - base: 100 - factor: 2 - jitter: true - max_delay: "5s" - max_attempts: 5 - - auth: - type: jwt - claims: - audience: ["${env:PEER_SPIFFE_ID}"] - issuer: "${env:SPIFFE_ID}" - subject: "${env:SPIFFE_ID}" - duration: "5m" - key: - type: encoding - algorithm: "RS256" - format: pem - key: - file: "/run/spire/jwt-signing-key.pem" - - controller: - servers: - - endpoint: "0.0.0.0:${env:CONTROLLER_PORT}" - tls: - insecure: false - source: - type: spire - socket_path: "/run/spire/sockets/agent.sock" - trust_domains: ["${env:SPIFFE_TRUST_DOMAIN}"] - client_ca: - type: spire - socket_path: "/run/spire/sockets/agent.sock" - trust_domains: ["${env:SPIFFE_TRUST_DOMAIN}"] - - auth: - type: jwt - claims: - audience: ["spiffe://${env:SPIFFE_TRUST_DOMAIN}/slim-controller"] - issuer: "spiffe://${env:SPIFFE_TRUST_DOMAIN}/slim-issuer" - subject: "${env:SPIFFE_ID}" - key: - type: decoding - algorithm: "RS256" - format: jwks - key: - file: "/run/spire/jwt-bundle.json" -``` - -## Configuration Reference Tables - -### Endpoint Configuration - -The `endpoint` field can be configured as either a network address or a Unix socket: - -- Network address: standard TCP address for gRPC/HTTP/2 connections (e.g., `0.0.0.0:8080`, `example.com:443`) -- Unix socket: local socket file path prefixed with `unix://` (e.g., `unix:///var/run/slim.sock`) - -!!! warning "Unix Socket Limitations" - When using Unix sockets, TLS and other transport-related options (such as `tls`, `keepalive`, `proxy`) are not supported and will be ignored. Unix sockets provide local inter-process communication without network transport. - -### Server Configuration Options - -| Field | Type | Required | Default | Description | Required When | -|-------|------|----------|---------|-------------|---------------| -| `endpoint` | string | ✅ | - | Listen address (network or unix socket) | Always | -| `tls.insecure` | boolean | ❌ | `false` | Disable TLS | - | -| `tls.source` | TlsSource | ⚠️ | `none` | Server certificate source | Required when `tls.insecure=false` | -| `tls.client_ca` | CaSource | ❌ | `none` | Client CA for mTLS | Optional (enables client cert verification) | -| `tls.tls_version` | string | ❌ | `"tls1.3"` | TLS protocol version | - | -| `http2_only` | boolean | ❌ | `true` | HTTP/2 only mode | - | -| `max_frame_size` | integer | ❌ | `4` | Max message size (MiB) | - | -| `max_concurrent_streams` | integer | ❌ | `100` | Max concurrent streams | - | -| `read_buffer_size` | integer | ❌ | `1048576` | Read buffer (bytes) | - | -| `write_buffer_size` | integer | ❌ | `1048576` | Write buffer (bytes) | - | -| `keepalive.max_connection_idle` | duration | ❌ | `"1h"` | Idle timeout | - | -| `keepalive.max_connection_age` | duration | ❌ | `"2h"` | Max connection age | - | -| `keepalive.time` | duration | ❌ | `"2m"` | Keepalive interval | - | -| `keepalive.timeout` | duration | ❌ | `"20s"` | Keepalive timeout | - | -| `auth` | AuthConfig | ❌ | `none` | Authentication config | - | -| `metadata` | object | ❌ | `null` | User metadata | - | - -### Client Configuration Options - -| Field | Type | Required | Default | Description | Required When | -|-------|------|----------|---------|-------------|---------------| -| `endpoint` | string | ✅ | - | Target endpoint (network or unix socket) | Always | -| `origin` | string | ❌ | `null` | Origin override | - | -| `server_name` | string | ❌ | `null` | SNI override | - | -| `tls.insecure` | boolean | ❌ | `false` | Disable TLS | - | -| `tls.insecure_skip_verify` | boolean | ❌ | `false` | Skip server verification | - | -| `tls.source` | TlsSource | ❌ | `none` | Client certificate | Optional (for mTLS) | -| `tls.ca_source` | CaSource | ❌ | `none` | Server CA verification | Optional (for server verification) | -| `tls.tls_version` | string | ❌ | `"tls1.3"` | TLS protocol version | - | -| `tls.include_system_ca_certs_pool` | boolean | ❌ | `true` | Include system CAs | - | -| `connect_timeout` | duration | ❌ | `"0s"` | Connection timeout | - | -| `request_timeout` | duration | ❌ | `"0s"` | Request timeout | - | -| `buffer_size` | integer | ❌ | `null` | Read buffer size | - | -| `headers` | map | ❌ | `{}` | Custom headers | - | -| `rate_limit` | string | ❌ | `null` | Rate limiting | - | -| `keepalive` | KeepaliveConfig | ❌ | `null` | Keepalive settings | - | -| `proxy` | ProxyConfig | ❌ | - | HTTP proxy config | - | -| `auth` | AuthConfig | ❌ | `none` | Authentication config | - | -| `backoff` | BackoffConfig | ❌ | exponential | Retry backoff | - | -| `metadata` | object | ❌ | `null` | User metadata | - | - -### Authentication Types - -| Type | Server | Client | Description | Required Fields | -|------|--------|--------|-------------|-----------------| -| `basic` | ✅ | ✅ | Username/password | `username`, `password` | -| `jwt` | ✅ | ✅ | Dynamic JWT generation | `key` (with `algorithm`, `format`, `key.file` or `key.data`) | -| `static_jwt` | ❌ | ✅ | Pre-generated JWT from file | `file` | -| `none` | ✅ | ✅ | No authentication | None | - -!!! note - SPIRE is not a separate authentication type. Instead, SPIRE provides authentication through the TLS layer and the JWT layer. - - SPIRE configuration is done in the `tls` section, not the `auth` section. See [TLS Configuration](#tls-configuration) and [Native SPIRE Integration](#native-spire-integration) for more information. - -**JWT Key Requirements:** - -- `key.type` - Required: `encoding`, `decoding`, or `autoresolve` -- `key.algorithm` - Required when `type` is `encoding` or `decoding` -- `key.format` - Required when `type` is `encoding` or `decoding` (values: `pem`, `jwk`, `jwks`) -- `key.key.file` or `key.key.data` - Required: one must be provided - -### TLS Source Types - -| Type | Required Fields | Optional Fields | Description | -|------|-----------------|-----------------|-------------| -| `file` | `cert`, `key` | - | Load certificates from files | -| `pem` | `cert`, `key` | - | Inline PEM certificate data | -| `spire` | - | `socket_path`, `jwt_audiences`, `target_spiffe_id`, `trust_domains` | SPIRE Workload API integration | -| `none` | - | - | No TLS source configured | - -**SPIRE Field Details:** - -- `socket_path` - Optional (defaults to `SPIFFE_ENDPOINT_SOCKET` env var) -- `jwt_audiences` - Optional (defaults to `["slim"]`) -- `target_spiffe_id` - Optional (for requesting specific SPIFFE ID) -- `trust_domains` - Optional (for X.509 bundle retrieval override) - -### CA Source Types - -| Type | Required Fields | Optional Fields | Description | -|------|-----------------|-----------------|-------------| -| `file` | `path` | - | Load CA certificates from file | -| `pem` | `data` | - | Inline PEM CA certificate data | -| `spire` | - | `socket_path`, `jwt_audiences`, `target_spiffe_id`, `trust_domains` | SPIRE trust bundle | -| `none` | - | - | No CA source configured | - -**SPIRE Field Details:** - -- `socket_path` - Optional (defaults to `SPIFFE_ENDPOINT_SOCKET` env var) -- `jwt_audiences` - Optional (defaults to `["slim"]`) -- `target_spiffe_id` - Optional (for requesting specific SPIFFE ID) -- `trust_domains` - Optional (for bundle retrieval override) diff --git a/docs/slim/slim-data-plane.md b/docs/slim/slim-data-plane.md deleted file mode 100644 index 4ab0af6..0000000 --- a/docs/slim/slim-data-plane.md +++ /dev/null @@ -1,82 +0,0 @@ -# SLIM Messaging Layer - -The [SLIM](overview.md) Messaging Layer implements an efficient message routing and -delivery system between applications. - -## Client and Channel Naming - -In SLIM, all endpoints are identified by a routable name. To send a message to a -specific client, an application sends the message to its unique name. Client -names follow this pattern: - -``` -org/namespace/service/client -``` - -- **Organization**: The first component identifies the organization that deploys - the application. -- **Namespace**: This can be used to segment client deployments. For example, - in a multi-region deployment, the namespace might indicate the application's - location. It can be adapted to fit other user requirements as well. -- **Service**: This component specifies the service exposed by the client. When - multiple instances of the same client are deployed (such as several pods in a - Kubernetes cluster), the organization, namespace, and service components - remain the same for all instances. -- **Client**: The final component is generated by SLIM and is a hash of the - client's identity (e.g., a hash of its public key). This uniquely identifies - each client instance. - -This naming structure supports both Anycast and Unicast message delivery: - -- **Anycast**: By specifying only the first three components, a message can be - sent to any one of the running clients sharing that name. -- **Unicast**: By including the fourth component, the message is delivered - directly to the specified client instance. - -This approach enables efficient client discovery. In fact, the message will be -delivered by the SLIM network to a client that is able to process it, even if the -real name of the client is unknown. Anycast forwarding is primarily used in -this discovery phase. - -In addition to client endpoints, SLIM allows messages to be sent to Channels. A -Channel acts as a message group, allowing multiple clients to connect and -receive all messages exchanged on that channel. The channel name follows the -same structure as client names, but the last component is left empty, as it is -shared among all connected clients. - -For further details, please refer to the [SLIM -Specification](https://datatracker.ietf.org/doc/draft-mpsb-agntcy-slim/). - -## SLIM Sessions Layer - -The SLIM platform includes a session layer that connects application frameworks -to the underlying SLIM messaging infrastructure. This layer provides a simple -interface that abstracts the complexity of secure messaging and message -distribution from the application. - -The session layer offers several functionalities: - -- **Security**: All messages in SLIM are encrypted by default using the [MLS - protocol](https://www.rfc-editor.org/rfc/rfc9420.html), which guarantees - end-to-end encryption even when messages traverse intermediate nodes where TLS - connections are terminated. The session layer is responsible for MLS group - creation and updates, as well as message encryption and decryption. -- **Channel Management**: The session layer enables clients to be invited to or - removed from a channel as needed. -- **Message Delivery**: The session layer abstracts message passing between - applications and the SLIM message distribution network. It handles message - formatting, routing, and delivery confirmation, while providing simple send - and receive primitives to applications. - -The session layer offers two primary APIs for establishing new sessions: - -- **Point-to-Point**: Facilitates point-to-point communication with a specific service - instance. This session performs a discovery phase to bind the session - to a single instance; all subsequent messages in the session are sent to that - same endpoint. - -- **Group**: Supports many-to-many communication over a named channel. -Every message sent to the channel is delivered to all current participants. - -For more information about each session type, see the -[SLIM session](./slim-session.md) documentation. diff --git a/docs/slim/slim-group-tutorial.md b/docs/slim/slim-group-tutorial.md deleted file mode 100644 index 520bac1..0000000 --- a/docs/slim/slim-group-tutorial.md +++ /dev/null @@ -1,801 +0,0 @@ -# SLIM Group Communication Tutorial - -This tutorial shows how to set up secure group communication using -SLIM. The group is created by defining a group session and inviting -participants. Messages are sent to a shared channel where every member can read -and write. All messages are end-to-end encrypted using the -[MLS protocol](https://datatracker.ietf.org/doc/html/rfc9420). This tutorial is -based on the -[group.py](https://github.com/agntcy/slim/tree/slim-v1.1.0/data-plane/bindings/python/examples/group.py) -example in the SLIM repo. - -## Key Features - -- **Name-based Addressing**: In SLIM, all endpoints (channels and clients) have - a name, and messages use a name-based addressing scheme for content routing. -- **Session Management**: Allows for the creation and management of sessions using - both the SLIM Python Bindings and the SLIM Controller. -- **Broadcast Messaging**: Facilitates broadcast messaging to multiple - subscribers. -- **End-to-End Encryption**: Ensures secure communication using the [MLS - protocol](https://datatracker.ietf.org/doc/html/rfc9420). - -## Configure Client Identity and Implement the SLIM App - -Every participant in a group requires a unique identity for authentication and for use by the MLS protocol. This section explains how to set up identity and create a SLIM application instance. - -### SLIM App - -Every SLIM application requires both a unique identity and an authentication mechanism. The identity is used for end-to-end encryption via the MLS protocol, while authentication verifies the application to the SLIM network. In this tutorial, we use shared secret authentication for simplicity. For more advanced authentication methods (JWT, SPIRE), see the [SLIM documentation](./slim-authentication.md). - -The example code provides a `create_local_app` helper function (from [common.py](https://github.com/agntcy/slim/tree/slim-v1.1.0/data-plane/bindings/python/examples/common.py)) that simplifies the app creation and connection process. - -The `create_local_app` function handles three main tasks: - -1. Initialize the Global Service: Sets up the SLIM runtime and global service instance -2. Create Authentication: Determines the authentication mode (SPIRE, JWT, or shared secret) based on configuration -3. Connect to SLIM Server: Establishes connection to the SLIM network - -Here's how the function works: - -```python -async def create_local_app(config: BaseConfig) -> tuple[slim_bindings.App, int]: - """ - Build a Slim application instance using the global service. - - Resolution precedence for auth: - 1. If SPIRE options provided -> SPIRE dynamic identity flow. - 2. Else if jwt + bundle + audience provided -> JWT/JWKS flow. - 3. Else -> shared secret (must be provided). - - Args: - config: BaseConfig instance containing all configuration. - - Returns: - tuple[App, int]: Slim application instance and connection ID. - """ - # Initialize tracing and global state - service = setup_service() - - # Convert local identifier to a strongly typed Name. - local_name = split_id(config.local) - - # Connect to SLIM server (returns connection ID) - client_config = slim_bindings.new_insecure_client_config(config.slim) - conn_id = await service.connect_async(client_config) - - # Determine authentication mode - auth_mode = config.get_auth_mode() - - if auth_mode == AuthMode.SPIRE: - print("Using SPIRE dynamic identity authentication.") - provider_config, verifier_config = spire_identity( - socket_path=config.spire_socket_path, - target_spiffe_id=config.spire_target_spiffe_id, - jwt_audiences=config.spire_jwt_audience, - ) - local_app = service.create_app(local_name, provider_config, verifier_config) - elif auth_mode == AuthMode.JWT: - print("Using JWT + JWKS authentication.") - # These should always be set if auth_mode is JWT - if not config.jwt or not config.spire_trust_bundle: - raise ValueError( - "JWT and SPIRE trust bundle are required for JWT auth mode" - ) - provider_config, verifier_config = jwt_identity( - config.jwt, - config.spire_trust_bundle, - str(local_name), - aud=config.audience, - ) - local_app = service.create_app(local_name, provider_config, verifier_config) - else: - print("Using shared-secret authentication.") - local_app = service.create_app_with_secret(local_name, config.shared_secret) - - # Provide feedback to user (instance numeric id). - format_message_print(f"{local_app.id()}", "Created app") - - # Subscribe to the local name - await local_app.subscribe_async(local_name, conn_id) - - return local_app, conn_id -``` - -### Key Authentication Options - -The following key authentication options are available: - -#### SPIRE (Recommended for production) - -Uses the SPIRE Workload API for dynamic identity. Requires a running SPIRE agent. See [SLIM documentation](./slim-authentication.md) for setup details. - -#### JWT/JWKS (Production) - -Uses static JWT files with JWKS for verification. Suitable for environments with an existing JWT infrastructure. - -#### Shared Secret (Development only) - -Simple symmetric key authentication. - -## Group Communication Using the Python Bindings - -Now that you know how to set up a SLIM application, we can see how to create a group where multiple participants can exchange messages. We start by showing how to create a group session using the Python bindings. - -In this setting, one participant acts as moderator: it creates the group session and invites participants by sending invitation control messages. A detailed description of group sessions and the invitation process is available in the [SLIM documentation](./slim-session.md). - -### Creating the Group Session and Inviting Members - -The creator of the group session invites other members to join the group. The -session will be identified by a unique session ID, and the group communication -will take place over a specific channel name. The session creator is responsible -for managing the session lifecycle, including creating, updating, and -terminating the session as needed. - -As each participant is provided with an identity, setting up MLS for end-to-end -encryption is straightforward: the session is created with the -`mls_enabled` flag set to `True`, which will enable the MLS protocol for the -session. This ensures that all messages exchanged within the session are -end-to-end encrypted, providing confidentiality and integrity for the group -communication. - -```python -# Create group session configuration -session_config = slim_bindings.SessionConfig( - session_type=slim_bindings.SessionType.GROUP, - enable_mls=enable_mls, # Enable Messaging Layer Security for end-to-end encrypted & authenticated group communication. - max_retries=5, # Max per-message resend attempts upon missing ack before reporting a delivery failure. - interval=datetime.timedelta(seconds=5), # Ack / delivery wait window; after this duration a retry is triggered (until max_retries). - metadata={}, -) - -# Create session - returns a SessionContext -session = local_app.create_session(session_config, chat_channel) -# Wait for session to be established -await session.completion.wait_async() -created_session = session.session - -# Invite each provided participant -for invite in invites: - invite_name = split_id(invite) - await local_app.set_route_async(invite_name, conn_id) - handle = await created_session.invite_async(invite_name) - await handle.wait_async() - print(f"{local} -> add {invite_name} to the group") -``` - -This code comes from the -[group.py](https://github.com/agntcy/slim/tree/slim-v1.1.0/data-plane/bindings/python/examples/group.py) -example. - -A new group session is created by calling `local_app.create_session(...)` which returns a `SessionContext` -object containing both the session and a completion handle. The completion handle must be awaited to ensure -the session is fully established. - -Key configuration parameters for `SessionConfig`: - -- `session_type`: Set to `SessionType.GROUP` for group/multicast sessions. -- `enable_mls`: Set to `True` to enable MLS for end-to-end encryption. -- `max_retries`: Maximum number of retransmission attempts (upon missing ack) before notifying the application of delivery failure. -- `interval`: Duration to wait for an acknowledgment; if the ack is not received in time, a retry is triggered. If omitted / None, the session is unreliable (no retry/ack flow). -- `metadata`: Optional key-value pairs for session metadata. - -After session creation, the moderator invites participants via `created_session.invite_async(invite_name)`. -Each invite call returns a completion handle that should be awaited to ensure the invitation completes. - -### Implement Participants and Receive Messages - -The group participants are implemented in a similar way, but they -do not create the session. They create the SLIM service instance and wait -for invites. Once they receive the invite, they can read and write on the shared channel. - -```python -async def receive_loop( - local_app: slim_bindings.App, - created_session: slim_bindings.Session | None, - session_ready: asyncio.Event, - shared_session_container: list, -): - """ - Receive messages for the bound session. - - Behavior: - * If not moderator: wait for a new group session (listen_for_session_async()). - * If moderator: reuse the created_session reference. - * Loop forever until cancellation or an error occurs. - """ - if created_session is None: - print_formatted_text("Waiting for session...", style=custom_style) - session = await local_app.listen_for_session_async(None) - else: - session = created_session - - # Make session available to other tasks - shared_session_container[0] = session - session_ready.set() - - # Get source and destination names for display - source_name = session.source() - - while True: - try: - # Await next inbound message from the group session. - # Returns a ReceivedMessage object with context and payload. - received_msg = await session.get_message_async( - timeout=datetime.timedelta(seconds=30) - ) - ctx = received_msg.context - payload = received_msg.payload - - # Display sender name and message - sender = ctx.source_name if hasattr(ctx, "source_name") else source_name - print_formatted_text( - f"{sender} > {payload.decode()}", - style=custom_style, - ) - - # if the message metadata contains PUBLISH_TO this message is a reply - # to a previous one. In this case we do not reply to avoid loops - if "PUBLISH_TO" not in ctx.metadata: - reply = f"message received by {source_name}" - await session.publish_to_async(ctx, reply.encode(), None, ctx.metadata) - except asyncio.CancelledError: - # Graceful shutdown path (ctrl-c or program exit). - break - except Exception as e: - # Break if session is closed, otherwise continue listening - if "session closed" in str(e).lower(): - break - continue -``` - -Each non-moderator participant listens for an incoming session using -`local_app.listen_for_session_async(None)`. The `None` parameter means wait indefinitely for a session. -This returns a session object containing metadata such as session ID, type, source name, and destination name. -The moderator already holds this information and therefore reuses the existing -`created_session` (see `session = created_session`). - -Once a session is established, the function retrieves the source name using `session.source()` for display purposes. -Participants (including the moderator) then call `received_msg = await session.get_message_async(timeout=...)` to receive -messages. The returned `ReceivedMessage` object has two attributes: `context` (with source, destination, message type, and metadata) -and `payload` (the raw message bytes). - -The function displays each received message showing the sender's name and payload. Additionally, it automatically sends -an acknowledgment reply using `session.publish_to_async(ctx, reply, None, metadata)`, unless the received message already -contains "PUBLISH_TO" in its metadata (indicating it's a reply), which prevents infinite reply loops. - -### Publish Messages to the Session - -All participants can publish messages on the shared channel: - -```python -async def keyboard_loop( - created_session: slim_bindings.Session, - session_ready: asyncio.Event, - shared_session_container: list[slim_bindings.Session], - local_app: slim_bindings.App, -): - """ - Interactive loop allowing participants to publish messages. - - Typing 'exit' or 'quit' (case-insensitive) terminates the loop. - Typing 'remove NAME' removes a participant from the group - Typing 'invite NAME' invites a participant to the group - Each line is published to the group channel as UTF-8 bytes. - """ - try: - # 1. Initialize an async session - prompt_session = PromptSession(style=custom_style) - - # Wait for the session to be established - await session_ready.wait() - - session = shared_session_container[0] - source_name = session.source() - dest_name = session.destination() - - if created_session: - print_formatted_text( - f"Welcome to the group {dest_name}!\n" - "Commands:\n" - " - Type a message to send it to the group\n" - " - 'remove NAME' to remove a participant\n" - " - 'invite NAME' to invite a participant\n" - " - 'exit' or 'quit' to leave the group", - style=custom_style, - ) - else: - print_formatted_text( - f"Welcome to the group {dest_name}!\n" - "Commands:\n" - " - Type a message to send it to the group\n" - " - 'exit' or 'quit' to leave the group", - style=custom_style, - ) - - while True: - # Run blocking input() in a worker thread so we do not block the event loop. - user_input = await prompt_session.prompt_async(f"{source_name} > ") - - if user_input.lower() in ("exit", "quit") and created_session: - # Delete the session - handle = await local_app.delete_session_async( - shared_session_container[0] - ) - await handle.wait_async() - break - - if user_input.lower().startswith("invite ") and created_session: - invite_id = user_input[7:].strip() # Skip "invite " (7 chars) - await handle_invite(shared_session_container[0], invite_id) - continue - - if user_input.lower().startswith("remove ") and created_session: - remove_id = user_input[7:].strip() # Skip "remove " (7 chars) - await handle_remove(shared_session_container[0], remove_id) - continue - - # Send message to the channel_name specified when creating the session. - # As the session is group, all participants will receive it. - await shared_session_container[0].publish_async( - user_input.encode(), None, None - ) - except KeyboardInterrupt: - # Handle Ctrl+C gracefully - pass - except asyncio.CancelledError: - # Handle task cancellation gracefully - pass - except Exception as e: - print_formatted_text(f"-> Error sending message: {e}") -``` - -Messages are sent using `session.publish_async(payload, payload_type, metadata)`. -The payload is the message bytes, while payload_type and metadata are optional. -There is no explicit destination because the group channel was fixed at session creation and delivery -fans out to all participants. - -The `keyboard_loop` function provides different interfaces depending on whether the application is a moderator or a regular participant: - -- Moderators (who created the session) have additional commands: - - - `invite NAME` - Dynamically invite a new participant to the group using the `handle_invite()` helper - - `remove NAME` - Remove a participant from the group using the `handle_remove()` helper - - `exit` or `quit` - Delete the session, which notifies all participants - -- Regular participants can: - - - Type messages to broadcast to the group - - `exit` or `quit` - Leave the group (local session only) - -When a user types `exit` or `quit` as a moderator, the application calls `local_app.delete_session_async()` -which returns a completion handle that must be awaited to ensure proper session cleanup before -terminating the loop. When the moderator closes the session, -all other participants are automatically notified, causing their receive loops to terminate -and their sessions to close gracefully. If the session closure is initiated by a participant, -only its local session is closed. - -### Run the Group Communication Example - -Now we will show how to run a new group session and -how to enable group communication on top of SLIM. The full code can be found in -[group.py](https://github.com/agntcy/slim/tree/slim-v1.1.0/data-plane/bindings/python/examples/group.py) -in the SLIM repo. To run the example, follow the steps listed here: - -#### Run SLIM - -As all members of the group are communicating via a SLIM network, we can set -up a SLIM instance representing the SLIM network. We use the pre-built -docker image for this purpose. - -First execute this command to create the SLIM configuration file. Details about -the configuration can be found in the SLIM repository documentation. - -```bash -cat << EOF > ./config.yaml -tracing: - log_level: info - display_thread_names: true - display_thread_ids: true - -runtime: - n_cores: 0 - thread_name: "slim-data-plane" - drain_timeout: 10s - -services: - slim/0: - dataplane: - servers: - - endpoint: "0.0.0.0:46357" - tls: - insecure: true - - clients: [] - controller: - servers: [] -EOF -``` - -This configuration starts a SLIM instance with a server listening on port -46357, without TLS encryption for simplicity. Messages are still encrypted -using the MLS protocol, but the connections between SLIM nodes do not use TLS. -In a production environment, it is recommended to always use TLS and configure -proper authentication and authorization mechanisms. - -You can run the SLIM instance using Docker: - -```bash -docker run -it \ - -v ./config.yaml:/config.yaml -p 46357:46357 \ - ghcr.io/agntcy/slim:1.0.0 /slim --config /config.yaml -``` - -If everything goes fine, you should see an output like this one: - -```bash -2026-01-28T15:03:38.408128Z INFO main ThreadId(01) application_lifecycle: slim: 52: Runtime started -2026-01-28T15:03:38.414090Z INFO main ThreadId(01) application_lifecycle: slim_service::service: 338: dataplane server started endpoint=0.0.0.0:46357 -2026-01-28T15:03:38.414813Z INFO main ThreadId(01) application_lifecycle: slim_service::service: 225: no controller configuration provided, skipping controller startup -2026-01-28T15:03:38.414823Z INFO main ThreadId(01) application_lifecycle: slim: 65: service started service=slim/0 -... -``` - -#### Start the Participants - -In this example, we use two participants: `agntcy/ns/client-1` and `agntcy/ns/client-2`. - -First, clone the SLIM repository and install the dependencies: - -```bash -git clone --branch slim-v1.1.0 https://github.com/agntcy/slim.git -cd slim/data-plane/bindings/python -task python:bindings:build -task python:bindings:install-examples -``` - -Now run these commands in two different terminals from the `slim/data-plane/bindings/python` directory: - -```bash -task python:example:group:client-1 -``` - -```bash -task python:example:group:client-2 -``` - -This starts two participants authenticated with a shared secret. -The output of these commands should look like this: - -```bash -2026-01-28T15:20:54.595284Z INFO slim slim_service::service: 402: client connected endpoint=http://127.0.0.1:46357 conn_id=0 -Using shared-secret authentication. -12628940569466571367 Created app -Waiting for session... -``` - -#### Create the Group - -Run the moderator application to create the session and invite the two -participants. In another terminal, from the `slim/data-plane/bindings/python` directory, run: - -```bash -task python:example:group:moderator -``` - -The result should look like: - -```bash -2026-01-28T15:22:08.907149Z INFO slim slim_service::service: 402: client connected endpoint=http://127.0.0.1:46357 conn_id=0 -Using shared-secret authentication. -2704185522498177164 Created app -Creating new group session (moderator)... agntcy/ns/moderator/ffffffffffffffff -agntcy/ns/moderator -> add agntcy/ns/client-1/ffffffffffffffff to the group -agntcy/ns/moderator -> add agntcy/ns/client-2/ffffffffffffffff to the group -Welcome to the group agntcy/ns/chat/ffffffffffffffff! -Commands: - - Type a message to send it to the group - - 'remove NAME' to remove a participant - - 'invite NAME' to invite a participant - - 'exit' or 'quit' to leave the group -agntcy/ns/moderator/25873267c342088c > -``` - -Now `client-1` and `client-2` are invited to the group, so on both of their terminals you should -be able to see a welcome message such as: - -```bash -Welcome to the group agntcy/ns/chat/ffffffffffffffff! -Commands: - - Type a message to send it to the group - - 'exit' or 'quit' to leave the group -agntcy/ns/client-1/af43028974930a67 > -``` - -At this point, you can write messages from any terminal and they will be received by all other group participants. - -Writing 'exit' or 'quit' from the moderator will close all the applications. -You can also remove and add participants by using the `remove` and `invite` commands from the moderator. - -## Group Communication Using the SLIM Controller - -Previously, we saw how to run group communication using the Python bindings with an in-application moderator. -This participant creates the group session and invites all other participants. -In this section, we describe how to create and orchestrate a group using the SLIM Controller, and we show how all -these functions can be delegated to the controller. We reuse the same group example code in this section as well. - -### Application Differences - -With the controller, you do not need to set up a moderator in your application. All participants can be run as we did for `client-1` and `client-2` in the previous examples. In code, this means you can avoid creating a new group session (using `local_app.create_session`) and the invitation loop. You only need to implement the `receive_loop` where the application waits for new sessions. This greatly simplifies your code. - -### Run the Group Communication Example - -Now we will show how to set up a group using the SLIM Controller. The reference code for the -application is still [group.py](https://github.com/agntcy/slim/tree/slim-v1.1.0/data-plane/bindings/python/examples/group.py). To run this example, follow the steps listed here. - -#### Run the SLIM Controller - -First, start the SLIM Controller. Full details are in the [SLIM Controller](./slim-controller.md) documentation; here we reproduce the minimal setup. Create a configuration file: - -```bash -cat << EOF > ./config-controller.yaml -northbound: - httpHost: 0.0.0.0 - httpPort: 50051 - logging: - level: INFO - -southbound: - httpHost: 0.0.0.0 - httpPort: 50052 - logging: - level: INFO - -reconciler: - maxRequeues: 15 - maxNumOfParallelReconciles: 1000 - -logging: - level: INFO - -database: - filePath: /db/controlplane.db -EOF -``` - -This config defines two APIs exposed by the controller: - -- Northbound API: used by an operator (e.g. via slimctl) to configure channels and participants, as well as the SLIM network. -- Southbound API: used by SLIM nodes to synchronize with the controller. - -Start the controller with Docker: - -```bash -docker run -it \ - -v ./config-controller.yaml:/config.yaml -v .:/db -p 50051:50051 -p 50052:50052 \ - ghcr.io/agntcy/slim/control-plane:1.0.0 --config /config.yaml -``` - -If everything goes fine, you should see an output like this: - -```bash -2026-01-28T15:26:53Z INF Starting Route Reconciler thread_name=reconciler -2026-01-28T15:26:53Z INF Northbound API Service is listening on [::]:50051 -2026-01-28T15:26:53Z INF Southbound API Service is Listening on [::]:50052 -``` - -#### Run the SLIM Node - -With the controller running, start a SLIM node configured to talk to it over the Southbound API. This node config includes two additional settings compared to the file from the previous section: - -- A controller client used to connect to the Southbound API running on port 50052. -- A shared secret token provider that will be used by the SLIM node to send messages over the SLIM network. As with the normal application, you can use a shared secret or a proper JWT. - -Create the `config-slim.yaml` for the node using the command below. We use the `host.docker.internal` endpoint to reach the controller from inside the Docker container via the host. - -```bash -cat << EOF > ./config-slim.yaml -tracing: - log_level: info - display_thread_names: true - display_thread_ids: true - -runtime: - n_cores: 0 - thread_name: "slim-data-plane" - drain_timeout: 10s - -services: - slim/0: - dataplane: - servers: - - endpoint: "0.0.0.0:46357" - tls: - insecure: true - - clients: [] - controller: - servers: [] - clients: - - endpoint: "http://host.docker.internal:50052" - tls: - insecure: true - token_provider: - type: shared_secret - id: "slim/0" - data: "very-long-shared-secret-value-0123456789abcdef" -EOF -``` - -This starts a SLIM node that connects to the controller: - -```bash -docker run -it \ - -v ./config-slim.yaml:/config.yaml -p 46357:46357 \ - ghcr.io/agntcy/slim:1.0.0 /slim --config /config.yaml -``` - -If everything goes fine, you should see an output like this one: - -```bash -2026-01-28T15:40:45.189063Z INFO main ThreadId(01) application_lifecycle: slim: 52: Runtime started -2026-01-28T15:40:45.189274Z INFO main ThreadId(01) application_lifecycle: slim_service::service: 338: dataplane server started endpoint=0.0.0.0:46357 -2026-01-28T15:40:45.189348Z INFO main ThreadId(01) application_lifecycle: slim_controller::service: 1634: connecting to control plane config.endpoint=http://host.docker.internal:50052 -2026-01-28T15:40:45.192773Z INFO slim-data-plane ThreadId(03) slim_controller::service: 1518: connected to control plane endpoint=http://host.docker.internal:50052 -2026-01-28T15:40:45.192777Z INFO main ThreadId(01) application_lifecycle: slim: 65: service started service=slim/0 -2026-01-28T15:40:45.196599Z INFO slim-data-plane ThreadId(04) slim_controller::service: 890: Processed ConfigurationCommand connections=0 subscriptions_to_set=0 subscriptions_to_del=0 -... -``` - -On the Controller side, you can see that the new node registers with the controller. The -output should be similar to this: - -```bash -2026-01-28T15:40:45Z INF Registering node with ID: slim/0 svc=southbound -2026-01-28T15:40:45Z INF Connection details: [endpoint: 192.168.65.1:46357] nodeID=slim/0 svc=southbound -2026-01-28T15:40:45Z INF Create generic routes for node node_id=slim/0 service=RouteService -2026-01-28T15:40:45Z INF Sending routes to registered node slim/0 node_id=slim/0 -2026-01-28T15:40:45Z INF Sending configuration command to registered node connections_count=0 message_id=95c75638-aa2d-4043-8f27-cb26f453716e node_id=slim/0 subscriptions_count=0 subscriptions_to_delete_count=0 -2026-01-28T15:40:45Z INF Configuration command processing completed node_id=slim/0 original_message_id=95c75638-aa2d-4043-8f27-cb26f453716e -``` - -#### Run the Participants - -Because the controller manages the group lifecycle, no participant needs to be designated as moderator in code. Every application instance just waits for a session invite. In three separate terminals, from the folder -`data-plane/bindings/python/examples` run: - -```bash -uv run slim-bindings-group \ - --local agntcy/ns/client-1 \ - --shared-secret "very-long-shared-secret-value-0123456789abcdef" -``` - -```bash -uv run slim-bindings-group \ - --local agntcy/ns/client-2 \ - --shared-secret "very-long-shared-secret-value-0123456789abcdef" -``` - -```bash -uv run slim-bindings-group \ - --local agntcy/ns/client-3 \ - --shared-secret "very-long-shared-secret-value-0123456789abcdef" -``` - -Each terminal should show output similar to: - -```bash -2026-01-28T15:44:59.125043Z INFO slim slim_service::service: 402: client connected endpoint=http://127.0.0.1:46357 conn_id=0 -Using shared-secret authentication. -10494544672403736104 Created app -Waiting for session.. -``` - -At this point all applications are waiting for a new session. - -#### Manage the Group with slimctl - -Use `slimctl` (see [SLIM Controller](./slim-controller.md)) to send administrative commands to the controller. - -First, you need to run `slimctl`. To install it see the [slimctl documentation](./slim-controller.md#installing-slimctl). - -To verify that `slimctl` was downloaded successfully, run the following command: - -```bash -slimctl version -``` - -##### Create the Group - -Select any running participant to be the initial member of the group. This participant acts as the logical -moderator of the channel, similar to the Python bindings example. However, you don't -need to handle this explicitly in the code. Run the following command to create the channel: - -```bash -slimctl controller channel create moderators=agntcy/ns/client-1/10494544672403736104 -``` - -The full name of the application can be taken from the output in the console. The value -`10494544672403736104` is the actual ID of the `client-1` application returned by -SLIM and is visible in the logs. In your case, this value will be different. - -The expected response from `slimctl` is: - -```bash -Received response: agntcy/ns/hDxc8CKpElJUfTTief -``` - -The value `agntcy/ns/hDxc8CKpElJUfTTief` is the channel (or group) identifier (name) that must be used in subsequent commands. - -On the application side, `client-1` was added to the session, so you should see -something like this: - -```bash -Welcome to the group agntcy/ns/hDxc8CKpElJUfTTief/ffffffffffffffff! -Commands: - - Type a message to send it to the group - - 'exit' or 'quit' to leave the group -agntcy/ns/client-1/91a41cc6ee17c628 > -``` - -##### Add Participants - -Now that the new group is created, add the additional participants `client-2` and `client-3` using the following `slimctl` commands: - -```bash -slimctl controller participant add -c agntcy/ns/xyIGhc2igNGmkeBDlZ agntcy/ns/client-2 -slimctl controller participant add -c agntcy/ns/xyIGhc2igNGmkeBDlZ agntcy/ns/client-3 -``` - -The expected `slimctl` output is: - -```bash -Adding participant to channel ID agntcy/ns/hDxc8CKpElJUfTTief: agntcy/ns/client-2 -Participant added successfully to channel ID agntcy/ns/hDxc8CKpElJUfTTief: agntcy/ns/client-2 -``` - -Now all the participants are part of the same group, and so each client log should show that the join was successful: - -```bash -Welcome to the group agntcy/ns/hDxc8CKpElJUfTTief/ffffffffffffffff! -Commands: - - Type a message to send it to the group - - 'exit' or 'quit' to leave the group -agntcy/ns/client-2/e9b95e5edaee3e2c > -``` - -At this point, every member can send messages, and they will be received by all the other participants. - -##### Remove a Participant - -To remove one of the participants from the channel, run the following command: - -```bash -slimctl controller participant delete -c agntcy/ns/xyIGhc2igNGmkeBDlZ agntcy/ns/client-3 -``` - -The `slimctl` expected output is this: - -```bash -Deleting participant from channel ID agntcy/ns/hDxc8CKpElJUfTTief: agntcy/ns/client-3 -Participant deleted successfully from channel ID agntcy/ns/hDxc8CKpElJUfTTief: agntcy/ns/client-3 -``` - -The application on `client-3` exits because the session related to the group was closed, which breaks the -receive loop in the Python code. - -##### Delete channel - -To delete the channel, run the following command: - -```bash -slimctl controller channel delete agntcy/ns/xyIGhc2igNGmkeBDlZ -``` - -The `slimctl` output is this: - -```bash -Channel deleted successfully with ID: agntcy/ns/hDxc8CKpElJUfTTief -``` - -All applications connected to the group stop as their receive loops terminate. diff --git a/docs/slim/slim-group.md b/docs/slim/slim-group.md deleted file mode 100644 index 23127a6..0000000 --- a/docs/slim/slim-group.md +++ /dev/null @@ -1,168 +0,0 @@ -# SLIM Group Creation and Management - -One of the key features of [SLIM](overview.md) is its support for secure group communication. -In SLIM, a group consists of multiple clients that communicate through a shared -channel. Each channel is identified by a unique name, as described in the [SLIM -Messaging Layer](slim-data-plane.md). When MLS is enabled, group -communication benefits from end-to-end encryption. - -This guide provides all the information you need to create and manage groups within a -SLIM network. A full tutorial with examples is available in -[Group Communication Tutorial](./slim-group-tutorial.md). - -## Creating Groups with SLIM Bindings - -This section shows how to use the SLIM bindings to create a group. -This requires a [group session](./slim-session.md#group-session). A group -session is a channel shared among multiple participants and used to -send messages to everyone. When a new participant wants to join the channel, -they must be invited by the channel creator. - -The channel creator can be part of the application and can either -actively participate in the communication process (possibly implementing some -of the application logic) or serve solely as a channel moderator. - -Group creation is available in both Python and Go bindings. This section provides -the basic steps to follow with Python code snippets for setting up a group session. - -For Go examples, see the [group example](https://github.com/agntcy/slim/tree/slim-v1.1.0/data-plane/bindings/go/examples/group/main.go). - -The full Python code is available in [group.py](https://github.com/agntcy/slim/tree/slim-v1.1.0/data-plane/bindings/python/examples/group.py). - -### Create the Channel - -The channel can be created with a group session, -which initializes the corresponding state in the SLIM session layer. -In a group session, communication between participants can be encrypted -end-to-end, enabling MLS. - -```python -# Create group session configuration -session_config = slim_bindings.SessionConfig( - session_type=slim_bindings.SessionType.GROUP, - enable_mls=enable_mls, # Enable Messaging Layer Security for end-to-end encrypted & authenticated group communication. - max_retries=5, # Max per-message resend attempts upon missing ack before reporting a delivery failure. - interval=datetime.timedelta(seconds=5), # Ack / delivery wait window; after this duration a retry is triggered (until max_retries). - metadata={}, -) - -# Create session - returns a tuple (SessionContext, CompletionHandle) -session = local_app.create_session(session_config, chat_channel) -# Wait for session to be established -await session.completion.wait_async() -created_session = session.session -``` - -### Invite Participants to the Channel - -Once the group session is created, new participants can be invited -to join. Not all participants need to be added at the beginning; -you can add them later, even after communication has started. - -```python -for invite in invites: - invite_name = split_id(invite) - await local_app.set_route_async(invite_name, conn_id) - handle = await created_session.invite_async(invite_name) # invite participant - await handle.wait_async() # await for the invite to be finished - print(f"{local} -> add {invite_name} to the group") -``` - -### Listen for New Sessions and Messages - -Participants that need to join the group start without a session and wait to be -invited. To wait for an invitation, the application calls `listen_for_session`. -When an invite message is received, a new session is created at the SLIM session layer, -and `listen_for_session` returns the metadata for the newly created session. - -```python -print_formatted_text("Waiting for session...", style=custom_style) -session = await local_app.listen_for_session_async(None) # timeout: datetime.timedelta | None = wait indefinitely -``` - -When a new session is available, the participant can start listening for messages: - -```python -while True: - try: - # Await next inbound message from the group session. - # Returns a ReceivedMessage object with context and payload. - received_msg = await session.get_message_async( - timeout=datetime.timedelta(seconds=30) - ) - ctx = received_msg.context - payload = received_msg.payload - - # Display sender name and message - sender = ctx.source_name if hasattr(ctx, "source_name") else source_name - print_formatted_text( - f"{sender} > {payload.decode()}", - style=custom_style, - ) -``` - -### Send Messages on a Channel - -Each participant can also send messages at any time to the new session, and each message will be delivered to all participants connected to the same channel. - -```python -# Send message to the channel_name specified when creating the session. -# As the session is group, all participants will receive it. -await shared_session_container[0].publish_async( - user_input.encode(), # payload: bytes - None, # payload_type: str | None - None # metadata: dict[str, str] | None -) -``` - -## Creating Groups with the SLIM Controller - -Another way to create a group in a SLIM network is to use the -[SLIM Controller](./slim-controller.md). For a complete description -on how to run it and the commands to use for the group creation and -management, please refer to the [Group Communication Tutorial](./slim-group-tutorial.md). -In this section, we list the `slimctl` commands to replicate -what we showed in the previous section. - -### Create the Channel - -First of all, you need to run the applications that you want to add to the group. -At that point, you can create the group by specifying the first participant in the -group. This will assign the role of moderator (like in the Python bindings examples), -but all the invites/removals will be done using the Controller and no action needs to be -performed in the application. - -To create the group, run: - -```bash -slimctl controller channel create moderators=agntcy/ns/client-1/10494544672403736104 -``` - -The outcome should be something similar to this: - -```bash -Received response: agntcy/ns/hDxc8CKpElJUfTTief -``` - -The name in the response is the name of the new channel created, with only one participant -added (e.g. `moderators=agntcy/ns/client-1/10494544672403736104`). - -### Invite Participants to the Channel - -Now that the channel is created, you can start to invite new participants. To do so, you can use -the following command: - -```bash -slimctl controller participant add -c agntcy/ns/hDxc8CKpElJUfTTief agntcy/ns/client-2 -``` - -The reply to the command should be similar to this: - -```bash -Adding participant to channel ID agntcy/ns/hDxc8CKpElJUfTTief: agntcy/ns/client-2 -Participant added successfully to channel ID agntcy/ns/hDxc8CKpElJUfTTief: agntcy/ns/client-2 -``` - -Now the channel has two participants that can start to communicate -over the shared channel `agntcy/ns/hDxc8CKpElJUfTTief`. Message reception and publishing -must be done within the application in the same way as shown in the previous section. diff --git a/docs/slim/slim-howto.md b/docs/slim/slim-howto.md deleted file mode 100644 index 91950fe..0000000 --- a/docs/slim/slim-howto.md +++ /dev/null @@ -1,436 +0,0 @@ -# Getting Started with SLIM - -## Installation - -SLIM is composed of multiple components, each with its own installation instructions. Choose the components you need based on your use case. - -### SLIM Node - -The SLIM Node is the core component that handles messaging operations. - -You can install the SLIM Node using Docker, Cargo, Helm, or the CLI binary. Choose the method that best fits your infrastructure. - -=== "Docker" - - Pull the SLIM container image and run it with a configuration file: - - ```bash - docker pull ghcr.io/agntcy/slim:1.0.0 - ``` - - Create a configuration file: - - ```yaml - # config.yaml - tracing: - log_level: info - display_thread_names: true - display_thread_ids: true - - runtime: - n_cores: 0 - thread_name: "slim-data-plane" - drain_timeout: 10s - - services: - slim/0: - dataplane: - servers: - - endpoint: "0.0.0.0:46357" - tls: - insecure: true - - clients: [] - ``` - - Run the container: - - ```bash - docker run -it \ - -v ./config.yaml:/config.yaml -p 46357:46357 \ - ghcr.io/agntcy/slim:1.0.0 /slim --config /config.yaml - ``` - -=== "Cargo" - - Install SLIM using Rust's package manager: - - ```bash - RUSTFLAGS="--cfg mls_build_async" cargo install agntcy-slim - ``` - - Create a configuration file: - - ```yaml - # config.yaml - tracing: - log_level: info - display_thread_names: true - display_thread_ids: true - - runtime: - n_cores: 0 - thread_name: "slim-data-plane" - drain_timeout: 10s - - services: - slim/0: - dataplane: - servers: - - endpoint: "0.0.0.0:46357" - tls: - insecure: true - - clients: [] - ``` - - Run SLIM: - - ```bash - ~/.cargo/bin/slim --config ./config.yaml - ``` - -=== "Helm" - - For Kubernetes deployments, use the official Helm chart: - - ```bash - helm pull oci://ghcr.io/agntcy/slim/helm/slim --version v1.1.0 - ``` - - !!! note "Configuration" - For detailed configuration options, see the [values.yaml](https://github.com/agntcy/slim/blob/slim-v1.1.0/charts/slim/values.yaml) in the repository. - -=== "CLI Binary" - - For local development and testing, use the `slimctl` binary. - - Install the `slimctl` binary following the [instructions below](#slimctl). - - === "Default Configuration" - - Run with default settings: - - ```bash - slimctl slim start - ``` - - === "Custom Configuration" - - Create a configuration file: - - ```yaml - # config.yaml - tracing: - log_level: info - display_thread_names: true - display_thread_ids: true - - runtime: - n_cores: 0 - thread_name: "slim-data-plane" - drain_timeout: 10s - - services: - slim/0: - dataplane: - servers: - - endpoint: "0.0.0.0:46357" - tls: - insecure: true - - clients: [] - ``` - - Start SLIM with the configuration: - - ```bash - slimctl slim start --config ./config.yaml - ``` - -For more configuration options, see the [SLIM Configuration reference](./slim-data-plane-config.md). - -### SLIM Controller - -The SLIM Controller manages SLIM Nodes and provides a user-friendly interface for configuration. - -=== "Docker" - - Pull the controller image: - - ```bash - docker pull ghcr.io/agntcy/slim/control-plane:1.0.0 - ``` - - Create a configuration file: - - ```yaml - # slim-control-plane.yaml - northbound: - httpHost: 0.0.0.0 - httpPort: 50051 - logging: - level: INFO - - southbound: - httpHost: 0.0.0.0 - httpPort: 50052 - logging: - level: INFO - - reconciler: - maxRequeues: 15 - maxNumOfParallelReconciles: 1000 - - logging: - level: INFO - - database: - filePath: /db/controlplane.db - ``` - - Run the controller: - - ```bash - docker run -it \ - -v ./slim-control-plane.yaml:/config.yaml -v .:/db \ - -p 50051:50051 -p 50052:50052 \ - ghcr.io/agntcy/slim/control-plane:1.0.0 \ - -config /config.yaml - ``` - -=== "Helm" - - For Kubernetes deployments: - - ```bash - helm pull oci://ghcr.io/agntcy/slim/helm/slim-control-plane --version v1.1.0 - ``` - -### SLIM Bindings - -Language bindings allow you to integrate SLIM with your applications. - -=== "Python" - - Install using pip: - - ```bash - pip install slim-bindings - ``` - - Or add to your `pyproject.toml`: - - ```toml - [project] - # ... - dependencies = ["slim-bindings~=1.0"] - ``` - - For more information on the SLIM bindings, see the [Messaging Layer Tutorial](./slim-data-plane.md) and the [Python Examples](https://github.com/agntcy/slim/tree/slim-v1.1.0/data-plane/bindings/python/examples). - -=== "Go" - - Install the Go bindings: - - ```bash - go get github.com/agntcy/slim-bindings-go@v1.1.0 - ``` - - Run the setup tool to install native libraries: - - ```bash - go run github.com/agntcy/slim-bindings-go/cmd/slim-bindings-setup - ``` - - Add to your `go.mod`: - - ```go - require github.com/agntcy/slim-bindings-go v1.1.0 - ``` - - !!! warning "C Compiler Required" - The Go bindings use native libraries via [CGO](https://pkg.go.dev/cmd/cgo), so you'll need a C compiler installed on your system. - - For more information on the Go bindings, see the [Go Examples](https://github.com/agntcy/slim/tree/slim-v1.1.0/data-plane/bindings/go/examples). - -=== "Kotlin" - - Add the Kotlin bindings to your Gradle project: - - === "Maven Central" - - Add to your `build.gradle.kts`: - - ```kotlin - dependencies { - implementation("io.agntcy.slim:slim-bindings-kotlin:1.2.0") - } - ``` - - `mavenCentral()` is the default repository in Gradle, so no additional repository configuration is needed. - - === "GitHub Packages" - - Add the GitHub Packages repository and dependency to your `build.gradle.kts`: - - ```kotlin - repositories { - maven { - url = uri("https://maven.pkg.github.com/agntcy/slim") - credentials { - username = project.findProperty("gpr.user") as String? ?: System.getenv("GITHUB_ACTOR") - password = project.findProperty("gpr.key") as String? ?: System.getenv("GITHUB_TOKEN") - } - } - } - dependencies { - implementation("io.agntcy.slim:slim-bindings-kotlin:1.2.0") - } - ``` - - !!! note "GitHub Token Required" - To use GitHub Packages, you need a personal access token with `read:packages` scope. Set `GITHUB_ACTOR` (your username) and `GITHUB_TOKEN` (your token) as environment variables, or use `gpr.user` and `gpr.key` in `gradle.properties`. - - !!! note "JDK 17+ Required" - The Kotlin bindings use [JNA](https://github.com/java-native-access/jna) for native library loading and require JDK 17 or higher. - - For more information on the Kotlin bindings, see the [Kotlin Examples](https://github.com/agntcy/slim/tree/slim-bindings-v1.2.0/data-plane/bindings/kotlin/examples). - -### Slimctl - -`slimctl` is a command-line tool for managing SLIM Nodes and Controllers. - -#### Installation - -Choose your platform: - -=== "macOS (Apple Silicon)" - - ```bash - curl -LO https://github.com/agntcy/slim/releases/download/slimctl-v1.2.0/slimctl_1.2.0_darwin_arm64.tar.gz - tar -xzf slimctl_1.2.0_darwin_arm64.tar.gz - sudo mv slimctl /usr/local/bin/slimctl - sudo chmod +x /usr/local/bin/slimctl - ``` - - !!! warning "macOS Security" - You may need to allow the binary to run if blocked by Gatekeeper: - - ```bash - sudo xattr -rd com.apple.quarantine /usr/local/bin/slimctl - ``` - - Alternatively, go to **System Settings > Privacy & Security** and allow the application when prompted. - -=== "macOS (Intel)" - - ```bash - curl -LO https://github.com/agntcy/slim/releases/download/slimctl-v1.2.0/slimctl_1.2.0_darwin_amd64.tar.gz - tar -xzf slimctl_1.2.0_darwin_amd64.tar.gz - sudo mv slimctl /usr/local/bin/slimctl - sudo chmod +x /usr/local/bin/slimctl - ``` - -=== "Linux (AMD64)" - - ```bash - curl -LO https://github.com/agntcy/slim/releases/download/slimctl-v1.2.0/slimctl_1.2.0_linux_amd64.tar.gz - tar -xzf slimctl_1.2.0_linux_amd64.tar.gz - sudo mv slimctl /usr/local/bin/slimctl - sudo chmod +x /usr/local/bin/slimctl - ``` - -=== "Linux (ARM64)" - - ```bash - curl -LO https://github.com/agntcy/slim/releases/download/slimctl-v1.2.0/slimctl_1.2.0_linux_arm64.tar.gz - tar -xzf slimctl_1.2.0_linux_arm64.tar.gz - sudo mv slimctl /usr/local/bin/slimctl - sudo chmod +x /usr/local/bin/slimctl - ``` - -=== "Windows (AMD64)" - - Download and extract the Windows binary: - - ```powershell - # Using PowerShell - Invoke-WebRequest -Uri "https://github.com/agntcy/slim/releases/download/slimctl-v1.2.0/slimctl-windows-amd64.zip" -OutFile "slimctl.zip" - Expand-Archive -Path "slimctl.zip" -DestinationPath "." - - # Move to a directory in your PATH (e.g., C:\Program Files\slimctl\) - # Or add the current directory to your PATH - ``` - - Alternatively, download directly from the [releases page](https://github.com/agntcy/slim/releases/download/slimctl-v1.2.0/slimctl-windows-amd64.zip). - -=== "Windows (ARM64)" - - Download and extract the Windows binary: - - ```powershell - # Using PowerShell - Invoke-WebRequest -Uri "https://github.com/agntcy/slim/releases/download/slimctl-v1.2.0/slimctl-windows-arm64.zip" -OutFile "slimctl.zip" - Expand-Archive -Path "slimctl.zip" -DestinationPath "." - - # Move to a directory in your PATH (e.g., C:\Program Files\slimctl\) - # Or add the current directory to your PATH - ``` - - Alternatively, download directly from the [releases page](https://github.com/agntcy/slim/releases/download/slimctl-v1.2.0/slimctl-windows-arm64.zip). - -Check the [slimctl documentation](./slim-controller.md) for additional installation methods. - -#### Verification - -Verify the installation: - -```bash -slimctl help -``` - -This should display help information and available commands. - -## Building from Source - -You can build SLIM from source. - -### Prerequisites - -Install the following tools on your system: - -- [Taskfile](https://taskfile.dev/) -- [Rust](https://rustup.rs/) -- [Go](https://go.dev/doc/install) - -### Building SLIM - -Once all prerequisites are installed, clone the repository and build the components: - -```bash -# Clone the SLIM repository -git clone https://github.com/agntcy/slim -cd slim - -# Build the data plane (Rust) -task data-plane:build - -# Build the control plane (Go) -task control-plane:build -``` - -For more information on the build system and development workflow, see the [SLIM repository](https://github.com/agntcy/slim). - -## Next Steps - -You've installed SLIM! Here's what to do next: - -1. Read the [messaging layer documentation](./slim-data-plane.md) -2. Explore the [example applications](https://github.com/agntcy/slim/tree/slim-v1.1.0/data-plane/bindings/) -3. Learn about [configuration options](./slim-data-plane-config.md) -4. Join us on [Slack](https://join.slack.com/t/agntcy/shared_invite/zt-3hb4p7bo0-5H2otGjxGt9OQ1g5jzK_GQ) - -## Need Help? - -If you get stuck, check the [detailed documentation](./overview.md), ask questions in our [community forums](https://join.slack.com/t/agntcy/shared_invite/zt-3hb4p7bo0-5H2otGjxGt9OQ1g5jzK_GQ), or report issues on [GitHub](https://github.com/agntcy/slim). diff --git a/docs/slim/slim-mcp.md b/docs/slim/slim-mcp.md deleted file mode 100644 index 88afbca..0000000 --- a/docs/slim/slim-mcp.md +++ /dev/null @@ -1,1034 +0,0 @@ -# SLIM and MCP Integration - -This tutorial demonstrates how to use SLIM to transport -MCP (Model Context Protocol) messages. SLIM offers two primary integration -options, depending on whether you're building a new system or integrating with -an existing MCP server: - -1. **Using SLIM as an MCP Custom Transport Protocol**: MCP is designed to support - multiple transport protocols, with SLIM now available as one of these options. - To implement SLIM as a custom transport, you can install the - [slim-mcp](https://github.com/agntcy/slim-mcp-python) Python package - through pip and integrate it directly into your application. This approach is - ideal for new systems where you control both client and server components, - providing native SLIM support for efficient MCP message transport. - -2. **Using SLIM with a Proxy Server**: If you have an existing MCP server running - that uses SSE (Server-Sent Events) for transport, you can integrate SLIM by - deploying the [SLIM-MCP proxy](https://github.com/agntcy/slim-mcp-rust), - written in Rust for high performance. This proxy handles translation between SLIM clients - and your SSE-based MCP server, allowing SLIM clients to connect seamlessly - without requiring modifications to your existing server, making it an - effective solution for established systems. - -This tutorial guides you through both integration methods. You'll learn how to -use SLIM as a custom transport for MCP and how to configure the proxy server to -enable SLIM support for an SSE-based MCP server. By the end, you'll have all the -necessary tools to integrate SLIM with MCP in a way that best fits your system's -architecture. - -## Using SLIM as an MCP Custom Transport Protocol - -In this section of the tutorial, we implement and deploy two sample -applications: - -- A [LlamaIndex agent](https://github.com/agntcy/slim-mcp-python/tree/v0.2.0/slim_mcp/examples/llamaindex_time_agent) -that communicates with an MCP server over SLIM to perform time queries and timezone conversions. -- An [MCP time - server](https://github.com/agntcy/slim-mcp-python/tree/v0.2.0/slim_mcp/examples/mcp_server_time) - that implements SLIM as its transport protocol and processes requests from the LlamaIndex agent. - -### Prerequisites - -- [UV](https://docs.astral.sh/uv/getting-started/installation/) - A Python - package installer and environment manager. -- [Docker](https://docs.docker.com/get-started/get-docker/) - For running the - SLIM instance. - -### Setting Up the SLIM Instance - -Since client and server communicate using SLIM, we first need to deploy -a SLIM instance. We are using a pre-built Docker image for this purpose. - -First, execute the following command to create a configuration file for SLIM: - -```bash -cat << EOF > ./config.yaml -tracing: - log_level: info - display_thread_names: true - display_thread_ids: true - -runtime: - n_cores: 0 - thread_name: "slim-data-plane" - drain_timeout: 10s - -services: - slim/0: - dataplane: - servers: - - endpoint: "0.0.0.0:46357" - tls: - insecure: true - - clients: [] - controller: - servers: [] -EOF -``` - -Now launch the SLIM instance using the just created configuration file: - -```bash -docker run -it \ - -v ./config.yaml:/config.yaml -p 46357:46357 \ - ghcr.io/agntcy/slim:1.0.0 /slim --config /config.yaml -``` - -This command deploys an SLIM instance that listens on port 46357 for incoming -connections. This instance serves as the communication backbone between our -client and server applications. - -### Implementing the MCP Server - -Next, we'll implement a simple MCP server that processes requests from the -LlamaIndex agent. This server demonstrates how to use SLIM as a custom -transport protocol. - -First, create a new directory for our MCP server project: - -```bash -mkdir -p mcp-server-time/src/mcp_server_time -cd mcp-server-time -``` - -Now, create a `pyproject.toml` file in the project root to define the project -dependencies: - -```toml -# pyproject.toml -[project] -name = "mcp-server-time" -version = "0.1.0" -description = "MCP server providing tools for time queries and timezone conversions" -requires-python = ">=3.11" -dependencies = ["mcp==1.6.0", "slim-mcp>=0.2.0", "click>=8.1.8"] - -[project.scripts] -mcp-server-time = "mcp_server_time:main" - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" -``` - -Next, let's implement the MCP server that handles time queries and timezone -conversions. This implementation is based on the [official MCP example -server](https://github.com/modelcontextprotocol/servers/tree/main/src/time), -modified to support both SLIM and SSE as transport protocols. - -Create the following files in your project directory: - -
- -src/mcp_server_time/__init__.py - -
- -```python -# src/mcp_server_time/__init__.py - -from .server import main - -if __name__ == "__main__": - main() -``` - -
- -
- -
- -src/mcp_server_time/server.py - -
- -```python -# src/mcp_server_time/server.py - -""" -MCP Time Server - A server implementation for time and timezone conversion functionality. - -This module provides tools for getting current time in different timezones and -converting times between timezones. -""" - -import json -import logging -from collections.abc import Sequence -from datetime import datetime, timedelta -from enum import Enum -from zoneinfo import ZoneInfo - -import click -import slim_bindings -from mcp import types -from mcp.server.lowlevel import Server -from mcp.shared.exceptions import McpError -from pydantic import BaseModel - -from slim_mcp import create_local_app, run_mcp_server -from slim_mcp.examples.click_types import ClientConfigType - -logger = logging.getLogger(__name__) - - -class TimeTools(str, Enum): - """Enumeration of available time-related tools.""" - - GET_CURRENT_TIME = "get_current_time" # Tool to get current time in a timezone - CONVERT_TIME = "convert_time" # Tool to convert time between timezones - - -class TimeResult(BaseModel): - """Model representing a time result with timezone information.""" - - timezone: str # IANA timezone name - datetime: str # ISO formatted datetime string - is_dst: bool # Whether the timezone is in daylight saving time - - -class TimeConversionResult(BaseModel): - """Model representing the result of a time conversion between timezones.""" - - source: TimeResult # Source timezone information - target: TimeResult # Target timezone information - time_difference: str # String representation of time difference (e.g., "+2.0h") - - -class TimeConversionInput(BaseModel): - """Model for time conversion input parameters.""" - - source_tz: str # Source timezone - time: str # Time to convert in HH:MM format - target_tz_list: list[str] # List of target timezones - - -def get_local_tz(local_tz_override: str | None = None) -> ZoneInfo: - """ - Get the local timezone information. - - Args: - local_tz_override: Optional timezone override string - - Returns: - ZoneInfo: The local timezone information - - Raises: - McpError: If timezone cannot be determined - """ - if local_tz_override: - return ZoneInfo(local_tz_override) - - # Get local timezone from datetime.now() - tzinfo = datetime.now().astimezone(tz=None).tzinfo - if tzinfo is not None: - return ZoneInfo(str(tzinfo)) - raise McpError( - types.ErrorData( - code=types.INTERNAL_ERROR, - message="Could not determine local timezone - tzinfo is None", - ) - ) - - -def get_zoneinfo(timezone_name: str) -> ZoneInfo: - """ - Get ZoneInfo object for a given timezone name. - - Args: - timezone_name: IANA timezone name - - Returns: - ZoneInfo: The timezone information - - Raises: - McpError: If timezone is invalid - """ - try: - return ZoneInfo(timezone_name) - except Exception as e: - raise McpError( - types.ErrorData( - code=types.INTERNAL_ERROR, - message=f"Invalid timezone: {str(e)}", - ) - ) - - -class TimeServer: - """Core time server implementation providing time-related functionality.""" - - def get_current_time(self, timezone_name: str) -> TimeResult: - """ - Get current time in specified timezone. - - Args: - timezone_name: IANA timezone name - - Returns: - TimeResult: Current time information in the specified timezone - """ - timezone = get_zoneinfo(timezone_name) - current_time = datetime.now(timezone) - - return TimeResult( - timezone=timezone_name, - datetime=current_time.isoformat(timespec="seconds"), - is_dst=bool(current_time.dst()), - ) - - def convert_time( - self, source_tz: str, time_str: str, target_tz: str - ) -> TimeConversionResult: - """ - Convert time between timezones. - - Args: - source_tz: Source timezone name - time_str: Time to convert in HH:MM format - target_tz: Target timezone name - - Returns: - TimeConversionResult: Converted time information - - Raises: - ValueError: If time format is invalid - """ - source_timezone = get_zoneinfo(source_tz) - target_timezone = get_zoneinfo(target_tz) - - try: - parsed_time = datetime.strptime(time_str, "%H:%M").time() - except ValueError: - raise ValueError("Invalid time format. Expected HH:MM [24-hour format]") - - # Create a datetime object for today with the specified time - now = datetime.now(source_timezone) - source_time = datetime( - now.year, - now.month, - now.day, - parsed_time.hour, - parsed_time.minute, - tzinfo=source_timezone, - ) - - # Convert to target timezone - target_time = source_time.astimezone(target_timezone) - - # Calculate time difference between timezones - source_offset = source_time.utcoffset() or timedelta() - target_offset = target_time.utcoffset() or timedelta() - hours_difference = (target_offset - source_offset).total_seconds() / 3600 - - # Format time difference string - if hours_difference.is_integer(): - time_diff_str = f"{hours_difference:+.1f}h" - else: - # For fractional hours like Nepal's UTC+5:45 - time_diff_str = f"{hours_difference:+.2f}".rstrip("0").rstrip(".") + "h" - - return TimeConversionResult( - source=TimeResult( - timezone=source_tz, - datetime=source_time.isoformat(timespec="seconds"), - is_dst=bool(source_time.dst()), - ), - target=TimeResult( - timezone=target_tz, - datetime=target_time.isoformat(timespec="seconds"), - is_dst=bool(target_time.dst()), - ), - time_difference=time_diff_str, - ) - - -class TimeServerApp: - """Main application class for the MCP Time Server.""" - - def __init__(self, local_timezone: str | None = None): - """ - Initialize the Time Server application. - - Args: - local_timezone: Optional override for local timezone - """ - self.app: Server = Server("mcp-time") - self.time_server = TimeServer() - self.local_tz = str(get_local_tz(local_timezone)) - self._setup_tools() - - def _setup_tools(self): - """Setup tool definitions and handlers for the MCP server.""" - - @self.app.list_tools() - async def list_tools() -> list[types.Tool]: - """ - List available time tools. - - Returns: - list[types.Tool]: List of available time-related tools - """ - return [ - types.Tool( - name=TimeTools.GET_CURRENT_TIME.value, - description="Get current time in a specific timezone", - inputSchema={ - "type": "object", - "properties": { - "timezone": { - "type": "string", - "description": f"IANA timezone name (e.g., 'America/New_York', 'Europe/London'). Use '{self.local_tz}' as local timezone if no timezone provided by the user.", - } - }, - "required": ["timezone"], - }, - ), - types.Tool( - name=TimeTools.CONVERT_TIME.value, - description="Convert time between timezones", - inputSchema={ - "type": "object", - "properties": { - "source_timezone": { - "type": "string", - "description": f"Source IANA timezone name (e.g., 'America/New_York', 'Europe/London'). Use '{self.local_tz}' as local timezone if no source timezone provided by the user.", - }, - "time": { - "type": "string", - "description": "Time to convert in 24-hour format (HH:MM)", - }, - "target_timezone": { - "type": "string", - "description": f"Target IANA timezone name (e.g., 'Asia/Tokyo', 'America/San_Francisco'). Use '{self.local_tz}' as local timezone if no target timezone provided by the user.", - }, - }, - "required": ["source_timezone", "time", "target_timezone"], - }, - ), - ] - - @self.app.call_tool() - async def call_tool( - name: str, arguments: dict - ) -> Sequence[types.TextContent | types.ImageContent | types.EmbeddedResource]: - """ - Handle tool calls for time queries. - - Args: - name: Name of the tool to call - arguments: Dictionary of tool arguments - - Returns: - Sequence of content types containing the tool response - - Raises: - ValueError: If tool name is unknown or arguments are invalid - """ - - result: TimeResult | TimeConversionResult - - try: - match name: - case TimeTools.GET_CURRENT_TIME.value: - timezone = arguments.get("timezone") - if not timezone: - raise ValueError("Missing required argument: timezone") - result = self.time_server.get_current_time(timezone) - - case TimeTools.CONVERT_TIME.value: - if not all( - k in arguments - for k in ["source_timezone", "time", "target_timezone"] - ): - raise ValueError("Missing required arguments") - result = self.time_server.convert_time( - arguments["source_timezone"], - arguments["time"], - arguments["target_timezone"], - ) - case _: - raise ValueError(f"Unknown tool: {name}") - - return [ - types.TextContent( - type="text", text=json.dumps(result.model_dump(), indent=2) - ) - ] - - except Exception as e: - raise ValueError(f"Error processing mcp-server-time query: {str(e)}") - - -async def serve_slim( - local_timezone: str | None = None, - organization: str = "org", - namespace: str = "ns", - mcp_server: str = "time-server", - config: slim_bindings.ClientConfig | None = None, -) -> None: - """ - Main server function that initializes and runs the time server using SLIM transport. - - Args: - local_timezone: Optional override for local timezone - organization: Organization name - namespace: Namespace name - mcp_server: MCP server name - config: Server configuration (ClientConfig object or None) - """ - # Create MCP app - time_app = TimeServerApp(local_timezone).app - - # Create SLIM app - server_name = slim_bindings.Name(organization, namespace, mcp_server) - slim_app, connection_id = await create_local_app(server_name, config) - - logger.info(f"Starting time server: {slim_app.id()}") - - # Run the MCP server - await run_mcp_server(slim_app, time_app) - - -def serve_sse( - local_timezone: str | None = None, - port: int = 8000, -) -> None: - """ - Main server function that initializes and runs the time server using SSE transport. - - Args: - local_timezone: Optional override for local timezone - port: Server listening port - """ - time_app = TimeServerApp(local_timezone) - - from mcp.server.sse import SseServerTransport - from starlette.applications import Starlette - from starlette.responses import Response - from starlette.routing import Mount, Route - - sse = SseServerTransport("/messages/") - - async def handle_sse(request): - async with sse.connect_sse( - request.scope, request.receive, request._send - ) as streams: - await time_app.app.run( - streams[0], streams[1], time_app.app.create_initialization_options() - ) - return Response() - - starlette_app = Starlette( - debug=True, - routes=[ - Route("/sse", endpoint=handle_sse, methods=["GET"]), - Mount("/messages/", app=sse.handle_post_message), - ], - ) - - import uvicorn - - uvicorn.run(starlette_app, host="0.0.0.0", port=port) - - -@click.command(context_settings={"auto_envvar_prefix": "MCP_TIME_SERVER"}) -@click.option( - "--local-timezone", type=str, help="Override local timezone", default=None -) -@click.option("--transport", default="slim", help="transport option: slim or sse") -@click.option( - "--port", - default="8000", - type=int, - help="listening port, used only with sse transport", -) -@click.option( - "--organization", - default="org", - help="server organization, used only with slim transport", -) -@click.option( - "--namespace", default="ns", help="server namespace, used only with slim transport" -) -@click.option( - "--mcp-server", - default="time-server", - help="server name, used only with slim transport", -) -@click.option( - "--config", - default={ - "endpoint": "http://127.0.0.1:46357", - "tls": { - "insecure": True, - }, - }, - type=ClientConfigType(), - help="slim server configuration, used only with slim transport", -) -def main(local_timezone, transport, port, organization, namespace, mcp_server, config): - """ - MCP Time Server - Time and timezone conversion functionality for MCP. - """ - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - ) - - if transport == "slim": - import asyncio - - asyncio.run( - serve_slim(local_timezone, organization, namespace, mcp_server, config) - ) - else: - serve_sse(local_timezone, port) -``` - -
- -
- -The core component of the server implementation is the `serve_slim` function. -This function establishes a connection with our SLIM instance using the new -`create_local_app` and `run_mcp_server` functions. It creates a SLIM application -with the specified server name and configuration, then runs the MCP server. - -External clients can address this server using the SLIM name -`org/ns/time-server`. - -```python -async def serve_slim( - local_timezone: str | None = None, - organization: str = "org", - namespace: str = "ns", - mcp_server: str = "time-server", - config: slim_bindings.ClientConfig | None = None, -) -> None: - """ - Main server function that initializes and runs the time server using SLIM transport. - - Args: - local_timezone: Optional override for local timezone - organization: Organization name - namespace: Namespace name - mcp_server: MCP server name - config: Server configuration (ClientConfig object or None) - """ - # Create MCP app - time_app = TimeServerApp(local_timezone).app - - # Create SLIM app - server_name = slim_bindings.Name(organization, namespace, mcp_server) - slim_app, connection_id = await create_local_app(server_name, config) - - logger.info(f"Starting time server: {slim_app.id()}") - - # Run the MCP server - await run_mcp_server(slim_app, time_app) -``` - -After implementing all the necessary files, your project structure looks -like this: - -```bash -mcp-server-time/ -├── src/ -│ └── mcp_server_time/ -│ ├── __init__.py -│ └── server.py -└── pyproject.toml -``` - -To launch the server and begin listening for incoming connections, navigate to -the project directory and run: - -```bash -uv run mcp-server-time --local-timezone Europe/London -``` - -### Implementing the LlamaIndex Agent -With our MCP server up and running, let's implement a LlamaIndex agent that -will interact with the server. This agent will send time queries and timezone -conversion requests to our MCP server using the SLIM transport protocol. - -First, create a new directory for our LlamaIndex agent project: - -```bash -mkdir -p llamaindex-time-agent/src/llamaindex_time_agent -cd llamaindex-time-agent -``` - -Next, create a `pyproject.toml` file to define the agent's dependencies: - -```toml -# pyproject.toml - -[project] -name = "llamaindex-time-agent" -version = "0.1.0" -description = "A llamaindex agent using MCP server over SLIM for time queries" -requires-python = ">=3.12" -dependencies = [ - "mcp==1.6.0", - "slim-mcp>=0.2.0", - "click>=8.1.8", - "llama-index>=0.12.29", - "llama-index-llms-azure-openai>=0.3.2", - "llama-index-llms-ollama>=0.5.4", - "llama-index-llms-openai-like>=0.3.4", - "llama-index-tools-mcp>=0.1.2", -] - -[project.scripts] -llamaindex-time-agent = "llamaindex_time_agent:main" - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" -``` - -Let's create the Python files for our LlamaIndex agent that will handle -time queries and timezone conversions. Create the following files in your -project directory: - -
-src/llamaindex_time_agent/__init__.py -
- -```python -# src/llamaindex_time_agent/__init__.py - -from .main import main - -if __name__ == "__main__": - main() -``` - -
- -
- -
-src/llamaindex_time_agent/main.py -
- -```python -# src/llamaindex_time_agent/main.py - -import asyncio -import logging - -import click -import slim_bindings -from dotenv import load_dotenv -from llama_index.core.agent.workflow import ReActAgent -from llama_index.llms.azure_openai import AzureOpenAI -from llama_index.llms.ollama import Ollama -from llama_index.tools.mcp import McpToolSpec -from mcp import ClientSession - -from slim_mcp import create_local_app, create_client_streams -from slim_mcp.examples.click_types import ClientConfigType - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -# load .env file -load_dotenv() - - -async def amain( - llm_type, llm_endpoint, llm_key, organization, namespace, mcp_server, city, config -): - if llm_type == "azure": - kwargs = { - "engine": "gpt-4o-mini", - "model": "gpt-4o-mini", - "is_chat_model": True, - "azure_endpoint": llm_endpoint, - "api_key": llm_key, - "api_version": "2024-08-01-preview", - } - llm = AzureOpenAI(**kwargs) - elif llm_type == "ollama": - kwargs = { - "model": "llama3.2", - } - llm = Ollama(**kwargs) - else: - raise Exception("LLM type must be azure or ollama") - - logger.info("Starting SLIM client") - - # Create SLIM app - client_name = slim_bindings.Name("org", "ns", "time-agent") - client_app, connection_id = await create_local_app(client_name, config) - - logger.info("SLIM App created") - - # Set route to destination if we have a connection - destination = slim_bindings.Name(organization, namespace, mcp_server) - if connection_id is not None: - await client_app.set_route_async(destination, connection_id) - - logger.info("SLIM route set") - - # Create MCP client session using standard transport pattern - async with create_client_streams(client_app, destination) as (read, write): - logger.info("Creating MCP client session") - async with ClientSession(read, write) as mcp_session: - logger.info("Creating MCP tool spec") - - await mcp_session.initialize() - - mcp_tool_spec = McpToolSpec( - client=mcp_session, - ) - - tools = await mcp_tool_spec.to_tool_list_async() - print(tools) - - agent = ReActAgent(llm=llm, tools=tools) - - response = await agent.run( - user_msg=f"What is the current time in {city}?", - ) - - print(response) - - -@click.command(context_settings={"auto_envvar_prefix": "TIME_AGENT"}) -@click.option("--llm-type", default="azure") -@click.option("--llm-endpoint", default=None) -@click.option("--llm-key", default=None) -@click.option("--mcp-server-organization", default="org") -@click.option("--mcp-server-namespace", default="ns") -@click.option("--mcp-server-name", default="time-server") -@click.option("--city", default="New York") -@click.option( - "--config", - default={ - "endpoint": "http://127.0.0.1:46357", - "tls": { - "insecure": True, - }, - }, - type=ClientConfigType(), -) -def main( - llm_type, - llm_endpoint, - llm_key, - mcp_server_organization, - mcp_server_namespace, - mcp_server_name, - city, - config, -): - try: - asyncio.run( - amain( - llm_type, - llm_endpoint, - llm_key, - mcp_server_organization, - mcp_server_namespace, - mcp_server_name, - city, - config, - ) - ) - except KeyboardInterrupt: - logger.info("Keyboard interrupt") - except Exception as e: - logger.error(f"Error: {e}") - raise e -``` - -
- -
- -The key component of the agent is the `amain` function, which handles: - -1. LLM configuration (Azure OpenAI or Ollama). -2. SLIM application creation using `create_local_app` to establish the client's identity. -3. Connection to the MCP server using `create_client_streams` to get read/write streams. -4. MCP session initialization using `ClientSession` with the established streams. -5. Tool setup and agent execution. - -The agent establishes its identity through the SLIM name `org/ns/time-agent`, -and connects to the MCP server identified by the SLIM name constructed from -the organization, namespace, and server name parameters. - -After implementing all the necessary files, your agent project structure should -look like this: - -```bash -llamaindex-time-agent/ -├── src/ -│ └── llamaindex_time_agent/ -│ ├── __init__.py -│ └── main.py -└── pyproject.toml -``` -To run the agent, navigate to the project directory and use one of the following -commands based on your preferred LLM: - -**Option 1: Using Azure OpenAI:** -```bash -uv run llamaindex-time-agent \ - --llm-type=azure \ - --llm-endpoint=${AZURE_OPENAI_ENDPOINT} \ - --llm-key=${AZURE_OPENAI_API_KEY} \ - --city 'New York' -``` - -**Option 2: Using Ollama (locally):** -```bash -uv run llamaindex-time-agent \ - --llm-type=ollama \ - --city 'New York' -``` - -The agent connects to the MCP server through SLIM, sends a time query for the -specified city, and display the response. - -## Using SLIM with a Proxy Server for SSE-based MCP Servers - -In this section, we demonstrate how to set up and configure the SLIM-MCP Proxy -Server. This proxy enables SLIM-based clients to communicate with existing MCP -servers that use SSE (Server-Sent Events) as their transport protocol. By -following these steps, you'll create a bridge between SLIM clients and SSE-based -MCP servers without modifying the servers themselves. - -### Setting Up the SLIM Node - -First, ensure you have a SLIM node running in your environment. If you haven't -already set one up, follow the instructions provided in the previous section to -[deploy a SLIM instance](#setting-up-the-slim-instance). - -### Running the MCP Server with SSE Transport - -Let's set up the time-server using the SSE transport protocol instead of -SLIM. The server implementation is the same as described in the previous section, -but we configure it to use SSE: - -```bash -uv run mcp-server-time --local-timezone Europe/London --transport sse -``` - -Once the server starts successfully, you should see logs similar to this: - -```bash -INFO: Started server process [27044] -INFO: Waiting for application startup. -INFO: Application startup complete. -INFO: Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) -``` - -At this point, your time-server is up and running with SSE transport. - -### Setting up the SLIM-MCP Proxy - -To enable SLIM clients to communicate with the SSE-based time-server, you'll need -to configure and run the SLIM-MCP Proxy Server. Follow these steps to set up a -local proxy instance: - -1. **Determine your local IP address** (works on both macOS and Linux): - ```bash - # For macOS - LOCAL_ADDRESS=$(ifconfig | grep --color=never "inet " | grep -v 127.0.0.1 | awk '{print $2}' | head -n 1) - - # For Linux - LOCAL_ADDRESS=$(ip addr show | grep --color=never "inet " | grep -v 127.0.0.1 | awk '{print $2}' | cut -d/ -f1 | head -n 1) - - # Verify the IP was found correctly - echo "Using local IP address: ${LOCAL_ADDRESS}" - ``` - > If the automatic detection doesn't work for your system, you can manually - > set your IP address: - > ```bash - > LOCAL_ADDRESS=192.168.1.10 # Replace with your actual local IP address - > ``` - -2. **Create the configuration file for the proxy**: - ```bash - cat << EOF > ./config-proxy.yaml - # SLIM-MCP Proxy Configuration - - # Tracing settings for log visibility - tracing: - log_level: info - display_thread_names: true - display_thread_ids: true - - # Runtime configuration - runtime: - n_cores: 0 - thread_name: "slim-data-plane" - drain_timeout: 10s - - # Service configuration for connecting to the SLIM node - services: - slim/0: - dataplane: - clients: - - endpoint: "http://${LOCAL_ADDRESS}:46357" - tls: - insecure: true - EOF - ``` - -3. **Run the proxy using Docker**: - ```bash - docker run -it \ - -v $(pwd)/config-proxy.yaml:/config-proxy.yaml \ - ghcr.io/agntcy/slim-mcp-rust/mcp-proxy:0.2.1 /slim-mcp-proxy \ - --config /config-proxy.yaml \ - --svc-name slim/0 \ - --name org/mcp/proxy \ - --mcp-server http://${LOCAL_ADDRESS}:8000/sse - ``` - This command: - - Mounts your local configuration file into the container. - - Uses the official SLIM-MCP proxy image from the [slim-mcp-rust](https://github.com/agntcy/slim-mcp-rust) repository. - - Sets the service name and proxy identifier. - - Configures the connection to your SSE-based MCP server. - -### Running the Agent with the Proxy - -Finally, you can run the LlamaIndex agent as shown in the previous section. The -agent automatically connects to the proxy, which then relays messages to -and from the MCP server. Notice that the proxy is reachable using the name -`org/mcp/proxy`: - -```bash -uv run llamaindex-time-agent \ - --llm-type=azure \ - --llm-endpoint=${AZURE_OPENAI_ENDPOINT} \ - --llm-key=${AZURE_OPENAI_API_KEY} \ - --city 'New York' \ - --mcp-server-organization "org" \ - --mcp-server-namespace "mcp" \ - --mcp-server-name "proxy" -``` - -With this setup, your SLIM client can now communicate seamlessly with the -SSE-based MCP server through the proxy, without requiring any changes to the -server implementation. diff --git a/docs/slim/slim-otel.md b/docs/slim/slim-otel.md deleted file mode 100644 index 7420ae9..0000000 --- a/docs/slim/slim-otel.md +++ /dev/null @@ -1,433 +0,0 @@ -# OpenTelemetry Integration with SLIM - -The SLIM OpenTelemetry [repository](https://github.com/agntcy/slim-otel) provides a custom receiver and exporter -that enable the standard OpenTelemetry Collector to send and receive observability -data over secure, low-latency SLIM channels. These components can be integrated -into any OpenTelemetry Collector pipeline alongside standard receivers, processors, -and exporters, enabling secure distribution of telemetry data (traces, metrics, -and logs) using SLIM's end-to-end encryption with Message Layer Security (MLS) -and flexible channel-based routing. - -## How It Works - -### SLIM Exporter - -The [SLIM exporter](https://github.com/agntcy/slim-otel/tree/v0.1.0/exporter/slimexporter) operates as follows: - -1. Connects to a SLIM node using the configured endpoint. -2. Registers three applications with the SLIM node (one for each signal type) - using the names specified in `exporter-names` configuration, making them - discoverable to other SLIM participants. -3. Creates channels based on the configuration, with each channel handling one - signal type (traces, metrics, or logs). -4. Invites participants to the created channels if configured -5. Publishes OpenTelemetry data (serialized as protobuf) to the appropriate SLIM - channels based on signal type. -6. Listens for invitations from other participants to join additional channels. - -The exporter can operate in multiple modes simultaneously: it can create and -manage its own channels while also accepting invitations to join channels created -by other participants. - -### SLIM Receiver - -The [SLIM receiver](https://github.com/agntcy/slim-otel/tree/v0.1.0/receiver/slimreceiver) operates as follows: - -1. Connects to a SLIM node using the configured endpoint. -2. Registers as an application with the configured `receiver-name`, making it - discoverable to other SLIM participants. -3. Listens for incoming SLIM sessions from any participant that wants to send - telemetry data. -4. Automatically detects the signal type (traces, metrics, or logs) by inspecting - the received data. -5. Supports multiple concurrent sessions from different senders simultaneously. - -## Getting Started - -This tutorial demonstrates how to set up and run the SLIM OpenTelemetry -Collector with a test application to send and receive telemetry data. - -### Prerequisites - -- [Docker](https://docs.docker.com/get-started/get-docker/) for running the - SLIM node. -- [Go](https://go.dev/doc/install) for building the collector and test - applications. -- [Task](https://taskfile.dev/installation/) for executing build - and run commands. - -### Setting Up the SLIM Node - -Since the collector and applications communicate using SLIM, first deploy a SLIM -node. Create a configuration file for the SLIM instance: - -```bash -cat << EOF > slim-test-config.yaml -tracing: - log_level: info - display_thread_names: true - display_thread_ids: true - -runtime: - n_cores: 0 - thread_name: "slim-data-plane" - drain_timeout: 10s - -services: - slim/0: - dataplane: - servers: - - endpoint: "0.0.0.0:46357" - tls: - insecure: true - clients: [] -EOF -``` - -Launch the SLIM node using Docker: - -```bash -docker run -it \ - -v ./slim-test-config.yaml:/config.yaml -p 46357:46357 \ - ghcr.io/agntcy/slim:1.0.0 /slim --config /config.yaml -``` - -The SLIM instance will listen on port 46357 for incoming connections, serving as -the communication backbone for the collector and applications. - -### Building the Collector - -The repository provides all the components and configuration needed to build a -custom OpenTelemetry Collector that includes the SLIM exporter and receiver. To build it: - -```bash -git clone --branch v0.1.0 https://github.com/agntcy/slim-otel.git -cd slim-otel -task collector:build -``` - -This command will: - -1. Download the [OpenTelemetry Collector Builder (OCB)](https://github.com/open-telemetry/opentelemetry-collector/tree/main/cmd/builder) if not already present -2. Generate collector sources based on `builder-config.yaml`, which includes the - SLIM components alongside standard OpenTelemetry components -3. Output the binary to `./slim-otelcol/slim-otelcol` - -The `builder-config.yaml` file specifies which components to include in the custom -collector. Here's the configuration from the repository: - -```yaml -dist: - name: slim-otelcol - description: Basic OTel Collector distribution for Developers - output_path: ./slim-otelcol - otelcol_version: 0.144.0 - -replaces: - - github.com/agntcy/slim/otel => .. - - github.com/agntcy/slim/otel/slimexporter => ../exporter/slimexporter - - github.com/agntcy/slim/otel/slimreceiver => ../receiver/slimreceiver - - github.com/agntcy/slim/otel/internal/sharedcomponent => ../internal/sharedcomponent - -exporters: - - gomod: github.com/agntcy/slim/otel/slimexporter v0.0.1 - - gomod: - go.opentelemetry.io/collector/exporter/debugexporter v0.144.0 - - gomod: - go.opentelemetry.io/collector/exporter/otlpexporter v0.144.0 - -processors: - - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.144.0 - -receivers: - - gomod: github.com/agntcy/slim/otel/slimreceiver v0.0.1 - - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.144.0 - -providers: - - gomod: go.opentelemetry.io/collector/confmap/provider/envprovider v1.48.0 - - gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v1.48.0 - - gomod: go.opentelemetry.io/collector/confmap/provider/httpprovider v1.48.0 - - gomod: go.opentelemetry.io/collector/confmap/provider/httpsprovider v1.48.0 - - gomod: go.opentelemetry.io/collector/confmap/provider/yamlprovider v1.48.0 -``` - -#### Using SLIM Components in Your Own Collector - -The SLIM exporter and receiver can be integrated into any OpenTelemetry Collector -distribution. Add them to your own `builder-config.yaml` to create a custom -collector that includes SLIM support alongside your other required components. - -For more information about the OpenTelemetry Collector Builder and standard -configurations, see the [OpenTelemetry Collector Builder documentation](https://opentelemetry.io/docs/collector/custom-collector/). - -### Configuring the Collector - -The collector uses a configuration file to define its behavior. You can configure -the SLIM exporter to send data over SLIM channels, the SLIM receiver to receive -data from SLIM channels, or both in the same collector instance. - -In this tutorial, we'll set up two collectors. The sender collector uses the SLIM exporter to send telemetry data over SLIM channels. The receiver collector uses the SLIM receiver to accept telemetry data from SLIM channels. - -The following diagram shows how these components interact: - -```mermaid -graph LR - OTLP[OTLP] -->|gRPC/HTTP| SC[Sender Collector
SLIM Exporter] - SC -->|SLIM Protocol| SN((SLIM Node
:46357)) - SN -->|SLIM Protocol| RC[Receiver Collector
SLIM Receiver] -``` - -The sender collector receives telemetry from applications (via OTLP). The -receiver collector outputs data via the debug exporter (visible in logs) and can -optionally forward to your observability backend. The SLIM node provides the secure -messaging layer between them. - -#### Configuring the Sender Collector - -Create the configuration file for the sender collector with the SLIM exporter: - -```bash -cat << EOF > sender-collector-config.yaml -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 - -processors: - batch: - timeout: 1s - send_batch_size: 1024 - -exporters: - slim: - endpoint: "http://127.0.0.1:46357" - exporter-names: - metrics: "agntcy/otel/exporter-metrics" - traces: "agntcy/otel/exporter-traces" - logs: "agntcy/otel/exporter-logs" - shared-secret: "a-very-long-shared-secret-0123456789-abcdefg" - channels: - - channel-name: "agntcy/otel/channel-traces" - signal: traces - participants: - - "agntcy/otel/receiver" - mls-enabled: true - - - channel-name: "agntcy/otel/channel-metrics" - signal: metrics - participants: - - "agntcy/otel/receiver" - mls-enabled: true - - - channel-name: "agntcy/otel/channel-logs" - signal: logs - participants: - - "agntcy/otel/receiver" - mls-enabled: true - -service: - telemetry: - metrics: - level: none - - pipelines: - traces: - receivers: [otlp] - processors: [batch] - exporters: [slim] - - metrics: - receivers: [otlp] - processors: [batch] - exporters: [slim] - - logs: - receivers: [otlp] - processors: [batch] - exporters: [slim] -EOF -``` - -To configure the SLIM exporter, there are required and optional settings. - -Required Settings: - -- `endpoint`: Address of the SLIM node (default: `http://127.0.0.1:46357`) -- `shared-secret`: Shared secret for MLS and identity provider authentication - -Optional Settings: - -- `exporter-names`: Names for each signal type exporter that identify this - collector instance in SLIM channels. - - For each signal type, the following options can be set: - - - `metrics`: Name for the metrics exporter (default: `agntcy/otel/exporter-metrics`). - - `traces`: Name for the traces exporter (default: `agntcy/otel/exporter-traces`). - - `logs`: Name for the logs exporter (default: `agntcy/otel/exporter-logs`). - -- `channels`: Array of channel configurations (default: `[]`). When empty, the - exporter operates in passive mode, only listening for invitations. - -Channels can also be configured. Each channel supports the following settings: - -- `channel-name` (required): The name of the SLIM channel in the form - `org/namespace/service`. -- `signal` (required): The signal type (`traces`, `metrics`, or `logs`). -- `participants` (required): Array of participant identifiers to invite. -- `mls-enabled` (default: `false`): Enable MLS encryption for the channel. - -#### Configuring the Receiver Collector - -Create the configuration file for the receiver collector that uses the SLIM receiver -to accept telemetry data: - -```bash -cat << EOF > receiver-collector-config.yaml -receivers: - slim: - endpoint: "http://127.0.0.1:46357" - receiver-name: "agntcy/otel/receiver" - shared-secret: "a-very-long-shared-secret-0123456789-abcdefg" - -processors: - batch: - timeout: 1s - send_batch_size: 1024 - -exporters: - debug: - verbosity: detailed - sampling_initial: 5 - sampling_thereafter: 200 - -service: - telemetry: - metrics: - level: none - - pipelines: - traces: - receivers: [slim] - processors: [batch] - exporters: [debug] - - metrics: - receivers: [slim] - processors: [batch] - exporters: [debug] - - logs: - receivers: [slim] - processors: [batch] - exporters: [debug] -EOF -``` - -To configure the SLIM receiver, there are required and optional settings. - -Required Settings: - -- `endpoint`: Address of the SLIM node (default: `http://127.0.0.1:46357`). -- `shared-secret`: Shared secret for MLS and identity provider authentication. - -Optional Settings: - -- `receiver-name`: Name for the receiver to be used in SLIM channels (default: - `agntcy/otel/receiver`). This is the identifier that other participants use to - establish sessions with this receiver. All signal types (traces, metrics, logs) - are handled by a single receiver instance. - -### Running the Collectors - -You can run each collector independently using the built binary and configuration files. - -In one terminal, start the receiver collector: - -```bash -./slim-otelcol/slim-otelcol --config receiver-collector-config.yaml -``` - -The receiver collector will: - -1. Connect to the SLIM node. -2. Register with the configured receiver name. -3. Listen for incoming sessions from the sender collector. - -In another terminal, start the sender collector: - -```bash -./slim-otelcol/slim-otelcol --config sender-collector-config.yaml -``` - -The sender collector will: - -1. Start listening for OTLP data on ports 4317 (gRPC) and 4318 (HTTP). -2. Connect to the SLIM node. -3. Create the configured channels and invite participants. - -Once the sender collector starts, you'll see the receiver collector accept the incoming sessions, one for each signal. - -``` -2026-01-29T18:57:44.070+0100 INFO slimreceiver@v0.0.1/receiver.go:97 New session received -2026-01-29T18:57:44.070+0100 INFO slimreceiver@v0.0.1/receiver.go:211 Handling new session {"sessionID": 1454232866, "sessionName": "agntcy/otel/channel-traces/ffffffffffffffff"} -2026-01-29T18:57:44.088+0100 INFO slimreceiver@v0.0.1/receiver.go:97 New session received -2026-01-29T18:57:44.088+0100 INFO slimreceiver@v0.0.1/receiver.go:211 Handling new session {"sessionID": 873926759, "sessionName": "agntcy/otel/channel-logs/ffffffffffffffff"} -2026-01-29T18:57:44.105+0100 INFO slimreceiver@v0.0.1/receiver.go:97 New session received -2026-01-29T18:57:44.105+0100 INFO slimreceiver@v0.0.1/receiver.go:211 Handling new session {"sessionID": 156147126, "sessionName": "agntcy/otel/channel-metrics/ffffffffffffffff"} -``` - -### Testing the Integration - -Now let's test the complete setup by running both collectors and sending telemetry -data through them using [telemetrygen](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/cmd/telemetrygen). - -Install [telemetrygen](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/cmd/telemetrygen) by running the following command: - -```bash -go install github.com/open-telemetry/opentelemetry-collector-contrib/cmd/telemetrygen@latest -``` - -Make sure the Go bin directory is in your PATH: - -```bash -export PATH=$PATH:$(go env GOPATH)/bin -``` - -Generate telemetry data and send it to the sender collector: - -**Generate traces:** - -```bash -telemetrygen traces --otlp-insecure --rate 10 --duration 30s -``` - -**Generate metrics:** - -```bash -telemetrygen metrics --otlp-insecure --rate 10 --duration 30s -``` - -**Generate logs:** - -```bash -telemetrygen logs --otlp-insecure --rate 10 --duration 30s -``` - -The telemetry data will flow through the architecture: - -**telemetrygen → Sender Collector → SLIM Node → Receiver Collector** - -You should see the telemetry data logged in the receiver collector's terminal output -via the debug exporter. - -## Additional Resources - -- [SLIM OpenTelemetry Collector Repository](https://github.com/agntcy/slim-otel) -- [SLIM Project](https://github.com/agntcy/slim) -- [SLIM Documentation](./overview.md) -- [OpenTelemetry Collector Documentation](https://opentelemetry.io/docs/collector/) -- [MLS RFC 9420](https://datatracker.ietf.org/doc/rfc9420/) diff --git a/docs/slim/slim-rpc.md b/docs/slim/slim-rpc.md deleted file mode 100644 index c2c78a8..0000000 --- a/docs/slim/slim-rpc.md +++ /dev/null @@ -1,598 +0,0 @@ -# SLIMRPC (SLIM Remote Procedure Call) - -SLIMRPC, or SLIM Remote Procedure Call, is a mechanism designed to enable -Protocol Buffers (protobuf) RPC over SLIM (Secure Low-latency Inter-process -Messaging). This is analogous to gRPC, which leverages HTTP/2 as its underlying -transport layer for protobuf RPC. - -A key advantage of SLIMRPC lies in its ability to seamlessly integrate SLIM as -the transport protocol for inter-application message exchange. This -significantly simplifies development: a protobuf file can be compiled to -generate code that utilizes SLIM for communication. Application developers can -then interact with the generated code much like they would with standard gRPC, -while benefiting from the inherent security features and efficiency provided by -the SLIM protocol. - -This documentation explains how SLIMRPC works and -how you can implement it in your applications. For detailed instructions on -compiling a protobuf file to obtain the necessary SLIMRPC stub code, please -refer to the dedicated [SLIMRPC compiler -documentation](./slim-slimrpc-compiler.md). - -## SLIM naming in SLIMRPC - -In SLIMRPC, each service and its individual RPC handlers are assigned a SLIM -name, facilitating efficient message routing and processing. Consider the -[example -protobuf](https://github.com/agntcy/slim/blob/slim-bindings-v1.1.1/data-plane/bindings/python/examples/slimrpc/simple/example.proto) -definition, which defines four distinct services: - -```proto -syntax = "proto3"; - -package example_service; - -service Test { - rpc ExampleUnaryUnary(ExampleRequest) returns (ExampleResponse); - rpc ExampleUnaryStream(ExampleRequest) returns (stream ExampleResponse); - rpc ExampleStreamUnary(stream ExampleRequest) returns (ExampleResponse); - rpc ExampleStreamStream(stream ExampleRequest) returns (stream ExampleResponse); -} - -message ExampleRequest { - string example_string = 1; - int64 example_integer = 2; -} - -message ExampleResponse { - string example_string = 1; - int64 example_integer = 2; -} -``` - -This example showcases the four primary communication patterns supported by -gRPC: - -- Unary-Unary -- Unary-Stream -- Stream-Unary -- Stream-Stream - -For SLIMRPC, a specific SLIM name is generated for each handler within a -service. This naming convention allows an application exposing the service to -listen for its name and process messages intended for a particular RPC method. The format -for these names is: - -``` -{package-name}.{service-name}-{handler_name} -``` - -Based on the example_service.Test definition, the names for each handler would -be: - -``` -example_service.Test-ExampleUnaryUnary -example_service.Test-ExampleUnaryStream -example_service.Test-ExampleStreamUnary -example_service.Test-ExampleStreamStream -``` - -This handler name is appended to the second component of the SLIM name -associated with the running application. For instance, to receive messages for -`example_service.Test-ExampleUnaryUnary`, an application would subscribe to: - -``` -component[0]/component[1]/component[2]-example_service.Test-ExampleUnaryUnary/component[3] -``` - -The subscription process is entirely managed by the SLIMRPC package. Application -developers are not required to explicitly handle SLIM name subscriptions. -Instead, they only need to implement the specific functions that will be invoked -when a message arrives for a defined RPC method, exactly as they would with -standard gRPC. - -## Example - -This section provides a detailed walkthrough of a basic SLIMRPC client-server -interaction in python, leveraging the simple example provided in -[example](https://github.com/agntcy/slim/blob/slim-bindings-v1.1.1/data-plane/bindings/python/examples/slimrpc/simple) -folder. - -### Generated Code - -The foundation of this example is the `example.proto` file, which is a standard -Protocol Buffers definition file. This file is compiled using the [SLIMRPC -compiler](./slim-slimrpc-compiler.md) to generate the necessary Python stub -code. The generated code is available in two files: -[example_pb2.py](https://github.com/agntcy/slim/blob/slim-bindings-v1.1.1/data-plane/bindings/python/examples/slimrpc/simple/types/example_pb2.py) -and -[example_pb2_slimrpc.py](https://github.com/agntcy/slim/blob/slim-bindings-v1.1.1/data-plane/bindings/python/examples/slimrpc/simple/types/example_pb2_slimrpc.py). -Specifically, `example_pb2_slimrpc.py` contains the SLIMRPC-specific stubs for -both client and server implementations. Below are the key classes and functions -generated by the compiler: - -_Client Stub (TestStub)_: The TestStub class represents the client-side -interface for interacting with the Test service. It provides methods for each -RPC defined in example.proto, allowing clients to initiate calls to the server. - -```python -class TestStub: - """Client stub for Test.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A slim_bindings.Channel. - """ - self._channel = channel - - async def ExampleUnaryUnary(self, request: pb2.ExampleRequest, timeout: Optional[timedelta] = None, metadata: Optional[dict[str, str]] = None) -> pb2.ExampleResponse: - """Call ExampleUnaryUnary method.""" - response_bytes = await self._channel.call_unary_async( - "example_service.Test", - "ExampleUnaryUnary", - pb2.ExampleRequest.SerializeToString(request), - timeout, - metadata, - ) - return pb2.ExampleResponse.FromString(response_bytes) - - async def ExampleUnaryStream(self, request: pb2.ExampleRequest, timeout: Optional[timedelta] = None, metadata: Optional[dict[str, str]] = None): - """Call ExampleUnaryStream method.""" - response_stream = await self._channel.call_unary_stream_async( - "example_service.Test", - "ExampleUnaryStream", - pb2.ExampleRequest.SerializeToString(request), - timeout, - metadata, - ) - while True: - stream_msg = await response_stream.next_async() - if stream_msg.is_end(): - break - if stream_msg.is_error(): - raise stream_msg[0] - if stream_msg.is_data(): - yield pb2.ExampleResponse.FromString(stream_msg[0]) - - async def ExampleStreamUnary(self, request_iterator, timeout: Optional[timedelta] = None, metadata: Optional[dict[str, str]] = None) -> pb2.ExampleResponse: - """Call ExampleStreamUnary method.""" - request_stream = self._channel.call_stream_unary( - "example_service.Test", - "ExampleStreamUnary", - timeout, - metadata, - ) - async for request in request_iterator: - await request_stream.send_async(pb2.ExampleRequest.SerializeToString(request)) - response_bytes = await request_stream.finalize_async() - return pb2.ExampleResponse.FromString(response_bytes) - - async def ExampleStreamStream(self, request_iterator, timeout: Optional[timedelta] = None, metadata: Optional[dict[str, str]] = None): - """Call ExampleStreamStream method.""" - bidi_stream = self._channel.call_stream_stream( - "example_service.Test", - "ExampleStreamStream", - timeout, - metadata, - ) - - async def send_requests(): - async for request in request_iterator: - await bidi_stream.send_async(pb2.ExampleRequest.SerializeToString(request)) - await bidi_stream.close_send_async() - - async def receive_responses(): - while True: - stream_msg = await bidi_stream.recv_async() - if stream_msg.is_end(): - break - if stream_msg.is_error(): - raise stream_msg[0] - if stream_msg.is_data(): - yield pb2.ExampleResponse.FromString(stream_msg[0]) - - # Start sending in background - import asyncio - send_task = asyncio.create_task(send_requests()) - - try: - async for response in receive_responses(): - yield response - finally: - await send_task -``` - -_Server Servicer (TestServicer)_: The TestServicer class defines the server-side -interface. Developers implement this class to provide the actual logic -for each RPC method. - -```python -class TestServicer: - """Server servicer for Test. Implement this class to provide your service logic.""" - - def ExampleUnaryUnary(self, request, context): - """Method for ExampleUnaryUnary. Implement your service logic here.""" - raise slim_bindings.RpcError.Rpc( - code=slim_bindings.RpcCode.UNIMPLEMENTED, - message="Method not implemented!", - details=None - ) - - def ExampleUnaryStream(self, request, context): - """Method for ExampleUnaryStream. Implement your service logic here.""" - raise slim_bindings.RpcError.Rpc( - code=slim_bindings.RpcCode.UNIMPLEMENTED, - message="Method not implemented!", - details=None - ) - - def ExampleStreamUnary(self, request_iterator, context): - """Method for ExampleStreamUnary. Implement your service logic here.""" - raise slim_bindings.RpcError.Rpc( - code=slim_bindings.RpcCode.UNIMPLEMENTED, - message="Method not implemented!", - details=None - ) - - def ExampleStreamStream(self, request_iterator, context): - """Method for ExampleStreamStream. Implement your service logic here.""" - raise slim_bindings.RpcError.Rpc( - code=slim_bindings.RpcCode.UNIMPLEMENTED, - message="Method not implemented!", - details=None - ) -``` - -_Handler Classes_: The generated code includes internal handler classes that wrap -each servicer method. These handlers manage request deserialization, response -serialization, and error handling. Below is an example of the unary-unary handler: - -```python -class _TestServicer_ExampleUnaryUnary_Handler: - def __init__(self, servicer): - self.servicer = servicer - - async def handle(self, request: bytes, context: slim_bindings.Context) -> bytes: - try: - request_msg = pb2.ExampleRequest.FromString(request) - response = await self.servicer.ExampleUnaryUnary(request_msg, context) - return pb2.ExampleResponse.SerializeToString(response) - except slim_bindings.RpcError: - raise - except Exception as e: - raise slim_bindings.RpcError.Rpc( - code=slim_bindings.RpcCode.INTERNAL, - message=str(e), - details=None - ) -``` - -Similar handler classes are generated for unary-stream, stream-unary, and -stream-stream patterns, each implementing the appropriate async handling logic -for their respective communication patterns. - -_Server Registration Function (add_TestServicer_to_server)_: This utility -function registers an implemented TestServicer instance with an SLIMRPC server. -It maps RPC method names to their corresponding handler instances. - -```python -def add_TestServicer_to_server(servicer, server: slim_bindings.Server): - server.register_unary_unary( - service_name="example_service.Test", - method_name="ExampleUnaryUnary", - handler=_TestServicer_ExampleUnaryUnary_Handler(servicer), - ) - server.register_unary_stream( - service_name="example_service.Test", - method_name="ExampleUnaryStream", - handler=_TestServicer_ExampleUnaryStream_Handler(servicer), - ) - server.register_stream_unary( - service_name="example_service.Test", - method_name="ExampleStreamUnary", - handler=_TestServicer_ExampleStreamUnary_Handler(servicer), - ) - server.register_stream_stream( - service_name="example_service.Test", - method_name="ExampleStreamStream", - handler=_TestServicer_ExampleStreamStream_Handler(servicer), - ) -``` - -### Server implementation - -The server-side logic is defined in -[server.py](https://github.com/agntcy/slim/blob/slim-bindings-v1.1.1/data-plane/bindings/python/examples/slimrpc/simple/server.py). -Similar to standard gRPC implementations, the core service functionality is -provided by the TestService class, which inherits from TestServicer (as -introduced in the previous section). This class contains the concrete -implementations for each of the defined RPC methods. - -The SLIM-specific code and configuration is handled within the amain() -asynchronous function: - -```python -async def amain() -> None: - slim_bindings.uniffi_set_event_loop(asyncio.get_running_loop()) - - # Initialize service - tracing_config = slim_bindings.new_tracing_config() - runtime_config = slim_bindings.new_runtime_config() - service_config = slim_bindings.new_service_config() - - tracing_config.log_level = "info" - - slim_bindings.initialize_with_configs( - tracing_config=tracing_config, - runtime_config=runtime_config, - service_config=[service_config], - ) - - service = slim_bindings.get_global_service() - - # Create local name - local_name = slim_bindings.Name("agntcy", "grpc", "server") - - # Connect to SLIM - client_config = slim_bindings.new_insecure_client_config("http://localhost:46357") - conn_id = await service.connect_async(client_config) - - # Create app with shared secret - local_app = service.create_app_with_secret( - local_name, "my_shared_secret_for_testing_purposes_only" - ) - - # Subscribe to local name - await local_app.subscribe_async(local_name, conn_id) - - # Create server - server = slim_bindings.Server.new_with_connection(local_app, local_name, conn_id) - - # Add servicer - add_TestServicer_to_server(TestService(), server) - - # Run server - await server.serve_async() -``` - -The server setup begins by setting the event loop for the SLIM bindings with -`uniffi_set_event_loop()`, which is required for proper async operation. The -service is then initialized with tracing, runtime, and service configurations. -The log level is set to "info" for appropriate logging verbosity. - -A local SLIM name is created using `slim_bindings.Name("agntcy", "grpc", -"server")`. This name is used to construct the full SLIMRPC names for each -method: - -``` -agntcy/grpc/server-example_service.Test-ExampleUnaryUnary -agntcy/grpc/server-example_service.Test-ExampleUnaryStream -agntcy/grpc/server-example_service.Test-ExampleStreamUnary -agntcy/grpc/server-example_service.Test-ExampleStreamStream -``` - -The server connects to a SLIM node running at `http://localhost:46357` using an -insecure client configuration (TLS disabled for simplicity in this example). A -local app is created with a shared secret for initializing the Message Layer -Security (MLS) protocol. - -After subscribing to the local name on the connection, a SLIMRPC server is -created with `slim_bindings.Server.new_with_connection()`. The -`add_TestServicer_to_server()` function is then called to register the -implemented `TestService` with the server, making its RPC methods available: - -```python - # Add servicer - add_TestServicer_to_server(TestService(), server) -``` - -Finally, the server starts serving requests with `await server.serve_async()`, -which runs indefinitely until interrupted. - -### Client implementation - -The client-side implementation, found in -[client.py](https://github.com/agntcy/slim/blob/slim-bindings-v1.1.1/data-plane/bindings/python/examples/slimrpc/simple/client.py), -largely mirrors the structure of a standard gRPC client. The client -initialization follows the same pattern as the server: - -```python -async def amain() -> None: - # Initialize service - tracing_config = slim_bindings.new_tracing_config() - runtime_config = slim_bindings.new_runtime_config() - service_config = slim_bindings.new_service_config() - - tracing_config.log_level = "info" - - slim_bindings.initialize_with_configs( - tracing_config=tracing_config, - runtime_config=runtime_config, - service_config=[service_config], - ) - - service = slim_bindings.get_global_service() - - # Create local and remote names - local_name = slim_bindings.Name("agntcy", "grpc", "client") - remote_name = slim_bindings.Name("agntcy", "grpc", "server") - - # Connect to SLIM - client_config = slim_bindings.new_insecure_client_config("http://localhost:46357") - conn_id = await service.connect_async(client_config) - - # Create app with shared secret - local_app = service.create_app_with_secret( - local_name, "my_shared_secret_for_testing_purposes_only" - ) - - # Subscribe to local name - await local_app.subscribe_async(local_name, conn_id) - - # Create channel - channel = slim_bindings.Channel.new_with_connection(local_app, remote_name, conn_id) - - # Create stubs - stubs = TestStub(channel) -``` - -The client initialization is very similar to the server setup. After initializing -the service with configurations, the client creates both a local name -(`agntcy/grpc/client`) and a remote name (`agntcy/grpc/server`). The local name -identifies the client in the SLIM network and is used by the server to send back -responses. - -Like the server, the client connects to the SLIM node at -`http://localhost:46357` using an insecure client configuration, creates an app -with a shared secret for MLS initialization, and subscribes to its local name. - -The key distinction from the server is the channel creation. The client uses -`slim_bindings.Channel.new_with_connection()` with the remote name -(`agntcy/grpc/server`) to establish a channel to the target server. This channel -is then used to create the `TestStub`, which provides type-safe access to all -the RPC methods defined in the protobuf. - -Once the stub is created, the client can invoke RPC methods using async/await -syntax: - -```python - # Unary-Unary call - request = ExampleRequest(example_integer=1, example_string="hello") - response = await stubs.ExampleUnaryUnary(request, timeout=timedelta(seconds=2)) - - # Unary-Stream call - async for resp in stubs.ExampleUnaryStream(request, timeout=timedelta(seconds=2)): - logger.info(f"Stream Response: {resp}") - - # Stream-Unary call - async def stream_requests() -> AsyncGenerator[ExampleRequest, None]: - for i in range(10): - yield ExampleRequest(example_integer=i, example_string=f"Request {i}") - - response = await stubs.ExampleStreamUnary( - stream_requests(), timeout=timedelta(seconds=2) - ) - - # Stream-Stream call - async for resp in stubs.ExampleStreamStream( - stream_requests(), timeout=timedelta(seconds=2) - ): - logger.info(f"Stream Stream Response: {resp}") -``` - -All RPC calls support timeout parameters and use Python's async/await patterns. -Streaming requests are provided as async generators, and streaming responses are -consumed using `async for` loops. - -## SLIMRPC under the Hood - -SLIMRPC was introduced to simplify the integration of existing applications with -SLIM. From a developer's perspective, using SLIMRPC or gRPC is almost identical. -Application developers do not need to manage endpoint names or connectivity -details, as these aspects are handled automatically by SLIMRPC and SLIM. - -All RPC services underneath utilize a point-to-point session. The SLIM -session creation is implemented inside SLIMRPC in -[channel.rs](https://github.com/agntcy/slim/blob/slim-bindings-v1.1.1/data-plane/bindings/rust/src/slimrpc/channel.rs#L657-L670): - -```rust -let slim_config = slim_session::session_config::SessionConfig { - session_type: ProtoSessionType::PointToPoint, - mls_enabled: true, - max_retries: Some(10), - interval: Some(Duration::from_secs(1)), - initiator: true, - metadata: ctx.metadata(), -}; - -// Create session to the method-specific subscription name -let (session_ctx, completion) = app - .create_session(slim_config, method_subscription_name.clone(), None) - .await - .map_err(|e| Status::unavailable(format!("Failed to create session: {}", e)))?; -``` - -The session used by SLIMRPC is also reliable. For each message, the sender -waits for an acknowledgment (ACK) packet for 1 second -(`timeout=datetime.timedelta(seconds=1)`). If no acknowledgment is received, the -message will be re-sent up to 10 times (`max_retries=10`) before notifying the -application of a communication error. - -All messages in a streaming communication will be -forwarded to the same application instance. Let's illustrate this with an -example using the client and server applications described above. - -Imagine two server instances running the same RPC service. In this example we will -focus on the Stream-Unary service, which is served by both server instances -under the general name -`agntcy/grpc/server-example_service.Test-ExampleStreamUnary`. In SLIM, each -application receives a unique ID. Thus, the full service name will include a -fourth component containing the server's ID. This ID is generated by SLIM itself -(see the [doc](./slim-data-plane.md) for more details). Here we will use -instance-1 and instance-2 for simplicity. So, the two full names for the -services will be: - -- `agntcy/grpc/server-example_service.Test-ExampleStreamUnary/instance-1` -- `agntcy/grpc/server-example_service.Test-ExampleStreamUnary/instance-2` - -Now, if a new client wants to use the Stream-Unary service it only needs to know -the general name -`agntcy/grpc/server-example_service.Test-ExampleStreamUnary`. SLIMRPC will -leverage SLIM's capabilities to first discover one of the available services, -and then SLIMRPC will use its full, specific name to consistently communicate -with that same endpoint. - -The client will register to SLIM with the name `agntcy/grpc/client`, and it will -get an unique ID assigned by SLIM, for example `instance-1`. So the full name of -the client will be `agntcy/grpc/client/instance-1`. - -```mermaid -sequenceDiagram - autonumber - - participant Client (instance-1) - participant SLIM Node - participant Server (instance-1) - participant Server (instance-2) - - - Note over Client (instance-1),Server (instance-1): Discovery - Client (instance-1)->>SLIM Node: Discover agntcy/grpc/server-example_service.Test-ExampleStreamUnary - SLIM Node->>Server (instance-1): Discover agntcy/grpc/server-example_service.Test-ExampleStreamUnary - Server (instance-1)->>SLIM Node: Ack from agntcy/grpc/server-example_service.Test-ExampleStreamUnary/instance-1 - SLIM Node->>Client (instance-1): Ack from agntcy/grpc/server-example_service.Test-ExampleStreamUnary/instance-1 - - Note over Client (instance-1),Server (instance-1): Stream - Client (instance-1)->>SLIM Node: Msg to agntcy/grpc/server-example_service.Test-ExampleStreamUnary/instance-1 - SLIM Node->>Server (instance-1): Msg to agntcy/grpc/server-example_service.Test-ExampleStreamUnary/instance-1 - Server (instance-1)->>SLIM Node: Ack from agntcy/grpc/server-example_service.Test-ExampleStreamUnary/instance-1 - SLIM Node->>Client (instance-1): Ack from agntcy/grpc/server-example_service.Test-ExampleStreamUnary/instance-1 - - Client (instance-1)->>SLIM Node: Msg to agntcy/grpc/server-example_service.Test-ExampleStreamUnary/instance-1 - SLIM Node->>Server (instance-1): Msg to agntcy/grpc/server-example_service.Test-ExampleStreamUnary/instance-1 - Server (instance-1)->>SLIM Node: Ack from agntcy/grpc/server-example_service.Test-ExampleStreamUnary/instance-1 - SLIM Node->>Client (instance-1): Ack from agntcy/grpc/server-example_service.Test-ExampleStreamUnary/instance-1 - - Note over Client (instance-1),Server (instance-1): Unary - Server (instance-1)->>SLIM Node: Msg to agntcy/grpc/client/instance-1 - SLIM Node->>Client (instance-1): Msg to agntcy/grpc/client/instance-1 - Client (instance-1)->>SLIM Node: Ack to agntcy/grpc/server-example_service.Test-ExampleStreamUnary/instance-1 - SLIM Node->>Server (instance-1): Ack to agntcy/grpc/server-example_service.Test-ExampleStreamUnary/instance-1 -``` - -The initial messages in the sequence diagram are used for the discovery phase. -After this step, the client application knows the specific name of the service -running on instance-1. It's important to note that the first message in the -discovery phase is sent in anycast from the SLIM node, meaning it could be -forwarded to **either of the two running servers**. For instance, a subsequent -call of the same RPC from the same client might be served by the server -with id `instance-2`. - -After the discovery, the client will always send messages to the same endpoint, -as demonstrated in the streaming session phase in the example. - -Finally, the server is expected to send one message to the client to close the -service. The server learns the client's address (where to forward the message) -by examining the source field of the received messages. diff --git a/docs/slim/slim-session.md b/docs/slim/slim-session.md deleted file mode 100644 index 0c6c436..0000000 --- a/docs/slim/slim-session.md +++ /dev/null @@ -1,398 +0,0 @@ -# SLIM Sessions - -This document explains the SLIM session layer and the two supported session -types. It helps you understand the two session interfaces, reliability, and security trade‑offs. - -The SLIM repository ships with practical, runnable examples for both [Python](https://github.com/agntcy/slim/tree/slim-v1.1.0/data-plane/bindings/python/examples) and [Go](https://github.com/agntcy/slim/tree/slim-v1.1.0/data-plane/bindings/go/examples) that demonstrate how to create sessions and exchange messages between applications. This document uses Python examples as reference. - -## Point-to-Point Session - -The point-to-point session enables point-to-point communication with a specific -instance. This session performs a discovery phase to bind to one instance and all -subsequent traffic in the session targets that same endpoint. With reliability -enabled, each message in the session must be acked. - -If MLS is enabled, the point-to-point session establishes a two‑member MLS group after -discovery. This mirrors the Group flow but with only two participants -(see [Group Session](#group-session)). - -The diagram below illustrates a point-to-point session from App-A to agntcy/ns/App-B. -App-A first discovers an available instance (App-B/1), then performs the MLS -setup, and finally sends multiple messages to that same instance, each followed -by an Ack. If MLS is not enabled, the MLS setup is skipped. - -```mermaid -sequenceDiagram - autonumber - - participant App-A - participant SLIM Node - participant App-B/1 - participant App-B/2 - - Note over App-A,App-B/2: Discovery - App-A->>App-A: Init MLS state - App-A->>SLIM Node: Discover agntcy/ns/App-B - SLIM Node->>App-B/1: Discover agntcy/ns/App-B - App-B/1->>SLIM Node: Discover Reply (agntcy/ns/App-B/1) - SLIM Node->>App-A: Discover Reply (agntcy/ns/App-B/1) - - Note over App-A,App-B/2: Invite - App-A->>SLIM Node: Invite agntcy/ns/App-B/1 - SLIM Node->>App-B/1: Invite agntcy/ns/App-B/1 - App-B/1->>App-B/1: Create new point-to-point Session - App-B/1->>SLIM Node: Invite Reply (MLS key package) - SLIM Node->>App-A: Invite Reply (MLS key package) - App-A->>App-A: Update MLS state - - Note over App-A,App-B/2: MLS setup - App-A->>SLIM Node: MLS Welcome agntcy/ns/App-B/1 - SLIM Node->>App-B/1: MLS Welcome agntcy/ns/App-B/1 - App-B/1->>App-B/1: Init MLS state - App-B/1->>SLIM Node: Ack(MLS Welcome) - SLIM Node->>App-A: Ack(MLS Welcome) - - Note over App-A,App-B/2: Message exchange - App-A->>SLIM Node: Message to agntcy/ns/App-B/1 - SLIM Node->>App-B/1: Message to agntcy/ns/App-B/1 - App-B/1->>SLIM Node: Ack - SLIM Node->>App-A: Ack - - App-A->>SLIM Node: Message to agntcy/ns/App-B/1 - SLIM Node->>App-B/1: Message to agntcy/ns/App-B/1 - App-B/1->>SLIM Node: Ack - SLIM Node->>App-A: Ack -``` - -### Create a Point-to-Point Session - -Using the SLIM Python bindings, you can create a point-to-point session as follows: - -```python -# Create session configuration -session_config = slim_bindings.SessionConfig( - session_type=slim_bindings.SessionType.POINT_TO_POINT, - enable_mls=enable_mls, - max_retries=5, - interval=datetime.timedelta(seconds=5), - metadata={}, -) - -# Create session - returns a context with completion and session -session_context = await local_app.create_session_async( - session_config, remote_name -) - -# Wait for session to be established -await session_context.completion.wait_async() - -session = session_context.session -``` - -Config Parameters: - -* `session_type` (required, SessionType): Either `POINT_TO_POINT` or `GROUP`. -* `enable_mls` (optional, bool): Enable end‑to‑end encryption (MLS). Default: `False`. -* `max_retries` (optional, int): Retry attempts per message if Ack missing. If not set, the session is best‑effort. -* `interval` (optional, timedelta): Wait per attempt for an Ack before retry. -* `metadata` (optional, dict): Custom metadata key-value pairs for the session. - -Create Session Parameters: - -* `session_config` (required, SessionConfig): The configuration object for this session. -* `remote_name` (required, Name): Identifier of the remote participant instance. - -The `await session_context.completion.wait_async()` guarantees that once returned, all the underlying message exchange -is done and the remote is correctly connected to the session. - -### Sending and Replying in a Point-to-Point Session - -As the point-to-point session is bound to a single remote instance after discovery, -outbound messages use the implicit destination. Use `publish` for normal sends -and to reply back to the sender. - -This example shows how to send and reply in a point-to-point session: - -```python -# Send a message using publish; it will reach the endpoint -# specified at session creation -await session.publish_async( - b"hello", # payload (bytes): message data to send - None, # payload_type (str|None): content type descriptor - None # metadata (dict|None): custom key-value pairs for message metadata -) - -# Await reply from remote (pattern depends on your control loop) -received_msg = await session.get_message_async( - timeout=datetime.timedelta(seconds=30) -) -reply = received_msg.payload -print(reply.decode()) - -# Send a correlated response back (echo style) -# The message will be sent according to the info in the session -await session.publish_async(reply, None, None) # payload, payload_type, metadata -``` - -### Point-to-Point Example - -This [example](https://github.com/agntcy/slim/blob/slim-v1.1.0/data-plane/bindings/python/examples/point_to_point.py) walks through the creation of a point-to-point session. When running the point-to-point example multiple times, the session binds to different running instances, while the message stream always sticks to the same endpoint. - -The example demonstrates how to publish messages, enable reliability, and enable MLS for end‑to‑end security. Run the example using the Taskfile provided in the repository. - -## Group Session - -The Group session allows many-to-many communication on a named channel. Each -message is delivered to all participants connected to the same session. - -The creator of the channel can act as a moderator, meaning that it can add or -remove participants from the session. Moderation can be built into your application -or delegated to a separate control service or the SLIM control plane. - -Below are examples using the latest Python bindings, along with explanations of -what happens inside the session layer when a participant is added or removed -from the channel (see [group management](./slim-group.md)). - -### Create a Group Session - -To create a group session, you need to configure the session with a topic -name and specify reliability and security settings. Here is an -example: - -```python -# Create group session configuration -session_config = slim_bindings.SessionConfig( - session_type=slim_bindings.SessionType.GROUP, - enable_mls=enable_mls, - max_retries=5, - interval=datetime.timedelta(seconds=5), - metadata={}, -) - -# Create session - returns a SessionContext -session = local_app.create_session(session_config, chat_channel) - -# Wait for session to be established -await session.completion.wait_async() - -created_session = session.session -``` - -Config Parameters: - -* `session_type` (required, SessionType): Set to `GROUP` for group sessions. -* `enable_mls` (optional, bool): Enable end‑to‑end encryption (MLS). Default: `False`. -* `max_retries` (optional, int): Retry attempts per message if Ack missing. If not set, the session is best‑effort. -* `interval` (optional, timedelta): Wait per attempt for an Ack before retry. -* `metadata` (optional, dict): Custom metadata key-value pairs for the session. - -Create Session Parameters: - -* `session_config` (required, SessionConfig): The configuration object for this session. -* `chat_channel` (required, Name): Identifier of the group channel that all participants join. - -As in the case of point-to-point sessions, the `await session.completion.wait_async()` guarantees that once returned, all the -underlying message exchange is completed. - -### Sending and Replying in a Group Session - -In a Group, the session targets a channel: all sends are delivered to all the current -participants. Use `publish_async` to send a message to all the participants in the group. - -```python -# Broadcast to the channel -await session.publish_async(b"hello", None, None) # payload, payload_type, metadata - -# Handle inbound messages -received_msg = await session.get_message_async( - timeout=datetime.timedelta(seconds=30) -) -ctx = received_msg.context -payload = received_msg.payload -print("channel received:", payload.decode()) -``` - -### Invite a New Participant - -The creator of the session can invite a new participant to the channel using the `invite` -method after creating the session. - -```python -# After creating the session: -for invite in invites: - invite_name = split_id(invite) - await local_app.set_route_async(invite_name, conn_id) - handle = await created_session.invite_async(invite_name) - await handle.wait_async() - print(f"{local} -> add {invite_name} to the group") -``` - -Parameters: - -* `invite_name` (required, Name): Identifier of the participant to add. - -Notice the `await local_app.set_route_async(invite_name, conn_id)` command before the invite. -This instructs SLIM on how to forward a message with the specified name. -This has to be done by the application for every invite. - -After `await handle.wait_async()` returns, you are guaranteed that the remote participant has been -correctly added to the group and all the state on every participant is updated. -When a moderator wants to add a new participant (e.g., an instance of App-C) to -a group session, all the following steps need to be executed (see diagram below): - -1. **Discovery Phase:** The moderator initiates a discovery request to find a - running instance of the desired application (App-C). This request is sent to - the SLIM Node, which forwards it via anycast to one of the App-C instances. - In the example, the message is forwarded to App-C/1 that replies with its - full identifier. The SLIM Node relays this reply back to the moderator. - -2. **Invitation:** The moderator sends an invite message for the discovered - instance (App-C/1) to the SLIM Node, which forwards it to App-C/1. Upon - receiving the invite, App-C/1 creates a new group session, subscribes to - the channel, and replies with its MLS (Messaging Layer Security) key - package. This reply is routed back to the moderator. - -3. **Group State Update:** The moderator initiates an MLS commit to add App-C/1 - to the secure group. The message is sent using the channel name and so the - SLIM Node distributes this commit to all current participants (App-B/2 and - App-A/1), who update their MLS state and acknowledge the commit. This message - also contains the new list of participants in the group, in particular the - newly added participant. The - moderator collects all acknowledgments for the update message. Once all acknowledgments are - received, the moderator sends an MLS Welcome message to App-C/1. App-C/1 - initializes its MLS state, gets the current list of participants in the group - and acknowledges receipt. At the end of this - process, all participants (including the new one) share a secure group state - and can exchange encrypted messages on the group channel. If MLS is - disabled, the update and welcome messages are used only to exchange the new list of - participants in the group, so that they are all aware of who is connected. - -```mermaid -sequenceDiagram - autonumber - - participant Moderator - participant SLIM Node - participant App-C/1 - participant App-C/2 - participant App-B/2 - participant App-A/1 - - Note over Moderator,App-A/1: Discovery - Moderator->>SLIM Node: Discover agntcy/ns/App-C - SLIM Node->>App-C/1: Discover agntcy/ns/App-C - App-C/1->>SLIM Node: Discover Reply from agntcy/ns/App-C/1 - SLIM Node->>Moderator: Discover Reply from agntcy/ns/App-C/1 - - Note over Moderator,App-A/1: Invite - Moderator->>SLIM Node: Invite agntcy/ns/App-C/1 - SLIM Node->>App-C/1: Invite agntcy/ns/App-C/1 - App-C/1->>App-C/1: Create new Group session - App-C/1->>SLIM Node: Subscribe to Channel - App-C/1->>SLIM Node: Invite Reply (MLS key package) - SLIM Node->>Moderator: Invite Reply (MLS key package) - Moderator->>Moderator: Update MLS state - - Note over Moderator,App-A/1: Group State Update - Moderator->>SLIM Node: Group Update (Add agntcy/ns/App-C/1, MLS commit) to Channel - par Group State Update on App-C/1 - SLIM Node->>App-B/2: Group Update (Add agntcy/ns/App-C/1, MLS commit) to Channel - App-B/2->>App-B/2: Update Group and MLS state - App-B/2->>SLIM Node: Ack(Group Update) - SLIM Node->>Moderator: Ack(Group Update) - and Group State Update on App-A/1 - SLIM Node->>App-A/1: Group Update (Add agntcy/ns/App-C/1, MLS commit) to Channel - App-A/1->>App-A/1: Update Group and MLS state - App-A/1->>SLIM Node: Ack(Group Update) - SLIM Node->>Moderator: Ack(Group Update) - end - Moderator->>SLIM Node: Welcome agntcy/ns/App-C/1 - SLIM Node->>App-C/1: Welcome agntcy/ns/App-C/1 - App-C/1->>App-C/1: Init Group and MLS state - App-C/1->>SLIM Node: Ack(Welcome) - SLIM Node->>Moderator: Ack(Welcome) -``` - -### Remove a Participant - -A moderator can remove a participant from the channel using the `remove` -method after creating the session. - -This example shows how to remove a participant from a group session: - -```python -# To remove a participant from the session: -await session.remove_async(remove_name) -``` - -Parameter: - -* `remove_name` (required, Name): Identifier of the participant to remove. - -When a moderator wants to remove a participant (e.g., App-C/1) from a group -session, the following steps occur. All the steps are visualized in the diagram -below: - -1. **Group State Update:** The moderator creates an MLS commit to remove App-C/1 - from the secure group. This commit is sent to the group channel and the - SLIM Node distributes it to all current participants (App-C/1, App-B/2, and - App-A/1). The message also contains the name of the participant to remove so - that all the endpoints can update their list of participants. - Each participant updates its MLS state and acknowledges the - commit. The moderator collects all acknowledgments. If MLS is - disabled, only the name of the participant to remove is sent using this message. - -2. **Removal:** After the MLS state is updated, the moderator sends a remove - message to App-C/1. Upon receiving the remove message, App-C/1 unsubscribes - from the channel, deletes its group session, and replies with a - confirmation. The SLIM Node relays this confirmation back to the moderator. - At the end of this process, App-C/1 is no longer a member of the group - and cannot send or receive messages on the channel. - -```mermaid -sequenceDiagram - autonumber - - participant Moderator - participant SLIM Node - participant App-C/1 - participant App-B/2 - participant App-A/1 - - Note over Moderator,App-A/1: Group State Update - Moderator->>SLIM Node: Group Update (Remove agntcy/ns/App-C/1, MLS commit) to Channel - par Handle Group State Update on App-C/1 - SLIM Node->>App-C/1: Group Update (Remove agntcy/ns/App-C/1, MLS commit) to Channel - App-C/1->>App-C/1: Update Group and MLS state - App-C/1->>SLIM Node: Ack(Group Update) - SLIM Node->>Moderator: Ack(Group Update) - and Handle Group State Update on App-B/2 - SLIM Node->>App-B/2: Group Update (Remove agntcy/ns/App-C/1, MLS commit) to Channel - App-B/2->>App-B/2: Update Group and MLS state - App-B/2->>SLIM Node: Ack(Group Update) - SLIM Node->>Moderator: Ack(Group Update) - and Handle Group State Update on App-A/1 - SLIM Node->>App-A/1: Group Update (Remove agntcy/ns/App-C/1, MLS commit) to Channel - App-A/1->>App-A/1: Update Group and MLS state - App-A/1->>SLIM Node: Ack(Group Update) - SLIM Node->>Moderator: Ack(Group Update) - end - - Note over Moderator,App-A/1: Remove - Moderator->>SLIM Node: Remove agntcy/ns/App-C/1 - SLIM Node->>App-C/1: Remove agntcy/ns/App-C/1 - App-C/1->>SLIM Node: Unsubscribe from Channel - App-C/1->>App-C/1: Remove Group session - App-C/1->>SLIM Node: Remove Reply - SLIM Node->>Moderator: Remove Reply -``` - -If the moderator is removed from the group, or it simply closes the session, it sends a -close message on the channel that is received by all the group participants. Upon reception, -all the participants acknowledge that the message was received and they close the local session, -similar to the reception of the remove message (see the diagram). In this way, when the moderator -stops, all participants are removed from the group. - -### Group Example - -This [example](https://github.com/agntcy/slim/blob/slim-v1.1.0/data-plane/bindings/python/examples/group.py) demonstrates how to create a group session, invite participants, and (if enabled) establish an MLS group for end-to-end encryption. It also shows how to broadcast messages to all current members and handle inbound group messages. Run the example using the Taskfile provided in the repository. diff --git a/docs/slim/slim-slimrpc-compiler.md b/docs/slim/slim-slimrpc-compiler.md deleted file mode 100644 index 166f6f5..0000000 --- a/docs/slim/slim-slimrpc-compiler.md +++ /dev/null @@ -1,365 +0,0 @@ -# SLIMRPC Compiler - -The Slim RPC Compiler is a collection of protoc plugins that generate client -stubs and server handlers for [SLIMRPC (Slim RPC)](./slim-rpc.md) from Protocol -Buffer service definitions. These plugins enable you to build high-performance -RPC services using the SLIMRPC framework. - -## Supported Languages - -- **Python**: `protoc-gen-slimrpc-python` -- **Go**: `protoc-gen-slimrpc-go` - -## Features - -The Slim RPC Compiler has the following features: - -- Generates type-safe client stubs and server handlers for SLIMRPC services. -- Supports all gRPC streaming patterns: unary-unary, unary-stream, stream-unary, - and stream-stream. -- Compatible with both `protoc` and `buf` build systems (buf recommended). -- Automatic import resolution for Protocol Buffer dependencies. - -## Installation - -You can install the SLIMRPC compiler either by downloading pre-built binaries or building from source using Cargo. - -=== "Pre-built Binaries (Python)" - - Download the pre-built binary for your platform from the [latest release](https://github.com/agntcy/slim/releases/tag/protoc-slimrpc-plugin-v1.0.2): - - === "Linux (x86_64)" - - ```bash - curl -LO https://github.com/agntcy/slim/releases/download/protoc-slimrpc-plugin-v1.0.2/protoc-gen-slimrpc-python-linux-x86_64.tar.gz - tar -xzf protoc-gen-slimrpc-python-linux-x86_64.tar.gz - chmod +x protoc-gen-slimrpc-python - sudo mv protoc-gen-slimrpc-python /usr/local/bin/ - ``` - - === "Linux (ARM64)" - - ```bash - curl -LO https://github.com/agntcy/slim/releases/download/protoc-slimrpc-plugin-v1.0.2/protoc-gen-slimrpc-python-linux-arm64.tar.gz - tar -xzf protoc-gen-slimrpc-python-linux-arm64.tar.gz - chmod +x protoc-gen-slimrpc-python - sudo mv protoc-gen-slimrpc-python /usr/local/bin/ - ``` - - === "macOS (ARM64)" - - ```bash - curl -LO https://github.com/agntcy/slim/releases/download/protoc-slimrpc-plugin-v1.0.2/protoc-gen-slimrpc-python-macos-arm64.tar.gz - tar -xzf protoc-gen-slimrpc-python-macos-arm64.tar.gz - chmod +x protoc-gen-slimrpc-python - sudo mv protoc-gen-slimrpc-python /usr/local/bin/ - ``` - - === "macOS (x86_64)" - - ```bash - curl -LO https://github.com/agntcy/slim/releases/download/protoc-slimrpc-plugin-v1.0.2/protoc-gen-slimrpc-python-macos-x86_64.tar.gz - tar -xzf protoc-gen-slimrpc-python-macos-x86_64.tar.gz - chmod +x protoc-gen-slimrpc-python - sudo mv protoc-gen-slimrpc-python /usr/local/bin/ - ``` - - === "Windows (x86_64)" - - ```powershell - Invoke-WebRequest -Uri "https://github.com/agntcy/slim/releases/download/protoc-slimrpc-plugin-v1.0.2/protoc-gen-slimrpc-python-windows-x86_64.zip" -OutFile "protoc-gen-slimrpc-python-windows-x86_64.zip" - Expand-Archive -Path protoc-gen-slimrpc-python-windows-x86_64.zip -DestinationPath . - # Add the binary to your PATH or move it to a directory in your PATH - ``` - - === "Windows (ARM64)" - - ```powershell - Invoke-WebRequest -Uri "https://github.com/agntcy/slim/releases/download/protoc-slimrpc-plugin-v1.0.2/protoc-gen-slimrpc-python-windows-arm64.zip" -OutFile "protoc-gen-slimrpc-python-windows-arm64.zip" - Expand-Archive -Path protoc-gen-slimrpc-python-windows-arm64.zip -DestinationPath . - # Add the binary to your PATH or move it to a directory in your PATH - ``` - -=== "Pre-built Binaries (Go)" - - Download the pre-built binary for your platform from the [latest release](https://github.com/agntcy/slim/releases/tag/protoc-slimrpc-plugin-v1.0.2): - - === "Linux (x86_64)" - - ```bash - curl -LO https://github.com/agntcy/slim/releases/download/protoc-slimrpc-plugin-v1.0.2/protoc-gen-slimrpc-go-linux-x86_64.tar.gz - tar -xzf protoc-gen-slimrpc-go-linux-x86_64.tar.gz - chmod +x protoc-gen-slimrpc-go - sudo mv protoc-gen-slimrpc-go /usr/local/bin/ - ``` - - === "Linux (ARM64)" - - ```bash - curl -LO https://github.com/agntcy/slim/releases/download/protoc-slimrpc-plugin-v1.0.2/protoc-gen-slimrpc-go-linux-arm64.tar.gz - tar -xzf protoc-gen-slimrpc-go-linux-arm64.tar.gz - chmod +x protoc-gen-slimrpc-go - sudo mv protoc-gen-slimrpc-go /usr/local/bin/ - ``` - - === "macOS (ARM64)" - - ```bash - curl -LO https://github.com/agntcy/slim/releases/download/protoc-slimrpc-plugin-v1.0.2/protoc-gen-slimrpc-go-macos-arm64.tar.gz - tar -xzf protoc-gen-slimrpc-go-macos-arm64.tar.gz - chmod +x protoc-gen-slimrpc-go - sudo mv protoc-gen-slimrpc-go /usr/local/bin/ - ``` - - === "macOS (x86_64)" - - ```bash - curl -LO https://github.com/agntcy/slim/releases/download/protoc-slimrpc-plugin-v1.0.2/protoc-gen-slimrpc-go-macos-x86_64.tar.gz - tar -xzf protoc-gen-slimrpc-go-macos-x86_64.tar.gz - chmod +x protoc-gen-slimrpc-go - sudo mv protoc-gen-slimrpc-go /usr/local/bin/ - ``` - - === "Windows (x86_64)" - - ```powershell - Invoke-WebRequest -Uri "https://github.com/agntcy/slim/releases/download/protoc-slimrpc-plugin-v1.0.2/protoc-gen-slimrpc-go-windows-x86_64.zip" -OutFile "protoc-gen-slimrpc-go-windows-x86_64.zip" - Expand-Archive -Path protoc-gen-slimrpc-go-windows-x86_64.zip -DestinationPath . - # Add the binary to your PATH or move it to a directory in your PATH - ``` - - === "Windows (ARM64)" - - ```powershell - Invoke-WebRequest -Uri "https://github.com/agntcy/slim/releases/download/protoc-slimrpc-plugin-v1.0.2/protoc-gen-slimrpc-go-windows-arm64.zip" -OutFile "protoc-gen-slimrpc-go-windows-arm64.zip" - Expand-Archive -Path protoc-gen-slimrpc-go-windows-arm64.zip -DestinationPath . - # Add the binary to your PATH or move it to a directory in your PATH - ``` - -=== "Build from Source (Cargo)" - - You can build and install the plugin from source using Cargo: - - ```bash - cargo install agntcy-protoc-slimrpc-plugin - ``` - - This will install the `protoc-slimrpc-plugin` binaries to your Cargo bin directory (usually `~/.cargo/bin`). - -## Usage - -### Example Protocol Buffer Definition - -Create a file called `example.proto`: - -```proto -syntax = "proto3"; - -package example_service; - -service Test { - rpc ExampleUnaryUnary(ExampleRequest) returns (ExampleResponse); - rpc ExampleUnaryStream(ExampleRequest) returns (stream ExampleResponse); - rpc ExampleStreamUnary(stream ExampleRequest) returns (ExampleResponse); - rpc ExampleStreamStream(stream ExampleRequest) returns (stream ExampleResponse); -} - -message ExampleRequest { - string example_string = 1; - int64 example_integer = 2; -} - -message ExampleResponse { - string example_string = 1; - int64 example_integer = 2; -} -``` - -If using Go, you might need to specify your Go package as well -as shown in [the simple example](https://github.com/agntcy/slim/blob/slim-bindings-v1.1.1/data-plane/bindings/go/examples/slimrpc/simple/example.proto). - -### Using with Buf (Recommended) - -#### Prerequisites - -- `buf` CLI [installed](https://buf.build/docs/cli/installation/) -- The appropriate `protoc-gen-slimrpc-python` or `protoc-gen-slimrpc-go` binary in your PATH, or specify the full path in the `buf.gen.yaml` file - -#### Create `buf.gen.yaml` - -Create a `buf.gen.yaml` file in your project root: - -=== "Python" - - ```yaml - version: v2 - managed: - enabled: true - inputs: - - proto_file: example.proto - plugins: - # Generate slimrpc stubs - - local: protoc-gen-slimrpc-python - out: types - # Generate standard protobuf code - - remote: buf.build/protocolbuffers/python:v29.3 - out: types - # Generate type stubs - - remote: buf.build/protocolbuffers/pyi:v31.1 - out: types - ``` - -=== "Go" - - ```yaml - version: v2 - managed: - enabled: true - plugins: - # Generate standard .pb.go files - - remote: buf.build/protocolbuffers/go - out: types - opt: - - paths=source_relative - # Generate slimrpc stubs - - local: protoc-gen-slimrpc-go - out: types - opt: - - paths=source_relative - ``` - -#### Generate Code - -```bash -buf generate -``` - -This will generate: - -- **Python**: `*_pb2.py` (protobuf types), `*_pb2.pyi` (type stubs), and `*_pb2_slimrpc.py` (slimrpc stubs) -- **Go**: `*.pb.go` (protobuf types) and `*_slimrpc.pb.go` (slimrpc stubs) - -#### Advanced buf Configuration (Python) - -You can customize the types import for Python. For example, to use existing -types from `a2a.grpc.a2a_pb2`, you can modify the `buf.gen.yaml` as follows: - -```yaml -version: v2 -managed: - enabled: true -plugins: - - local: protoc-gen-slimrpc-python - out: generated - opt: - - types_import=from a2a.grpc import a2a_pb2 as a2a__pb2 - - remote: buf.build/protocolbuffers/python - out: generated -``` - -### Using with Protocol Buffer Compiler - -#### Prerequisites - -Make sure you have: - -- `protoc` (Protocol Buffer compiler) installed -- The appropriate `protoc-gen-slimrpc-python` or `protoc-gen-slimrpc-go` binary in your PATH or specify its full path - -#### Generate Files - -=== "Python" - - ```bash - # Generate both the protobuf Python files and SLIMRPC files - protoc \ - --python_out=. \ - --pyi_out=. \ - --plugin=protoc-gen-slimrpc-python=/usr/local/bin/protoc-gen-slimrpc-python \ - --slimrpc-python_out=. \ - example.proto - ``` - - This will generate: - - - `example_pb2.py` - Standard protobuf Python bindings - - `example_pb2_slimrpc.py` - SLIMRPC client stubs and server servicers - -=== "Go" - - ```bash - # Generate both the protobuf Go files and SLIMRPC files - protoc \ - --go_out=. \ - --plugin=protoc-gen-slimrpc-go=/usr/local/bin/protoc-gen-slimrpc-go \ - --slimrpc-go_out=. \ - example.proto - ``` - - This will generate: - - - `example.pb.go` - Standard protobuf Go bindings - - `example_slimrpc.pb.go` - SLIMRPC client stubs and server servicers - -#### With Custom Types Import (Python) - -You can specify a custom import for the types module. This allows you to import -the types from an external package. - -For instance, if you don't want to generate the types and you want to import -them from `a2a.grpc.a2a_pb2`, you can do: - -```bash -protoc \ - --plugin=protoc-gen-slimrpc-python=/usr/local/bin/protoc-gen-slimrpc-python \ - --slimrpc-python_out=types_import="from a2a.grpc import a2a_pb2 as a2a__pb2":. \ - example.proto -``` - -## Examples - -Complete working examples are available in the repository: - -- **Python**: [bindings/python/examples/slimrpc/simple](https://github.com/agntcy/slim/tree/slim-bindings-v1.1.1/data-plane/bindings/python/examples/slimrpc/simple) -- **Go**: [bindings/go/examples/slimrpc/simple](https://github.com/agntcy/slim/tree/slim-bindings-v1.1.1/data-plane/bindings/go/examples/slimrpc/simple) - -Both examples demonstrate all four RPC patterns with comprehensive client and server implementations. - -## Generated Code Structure - -For detailed information about the generated code structure, including client stubs, server handlers, and registration functions for both Python and Go, please refer to the [SLIMRPC documentation](./slim-rpc.md#generated-code). - -The compiler generates type-safe client stubs and server handlers that support all gRPC streaming patterns (unary-unary, unary-stream, stream-unary, and stream-stream) with language-specific idioms and async/await patterns. - -## Plugin Parameters - -### Python Plugin - -- `types_import`: Customize how protobuf types are imported - - Example: `types_import="from my_package import types_pb2 as pb2"` - - Default: Uses local import based on the proto file name - -### Go Plugin - -- `paths`: Control output path strategy - - `source_relative`: Generate files relative to the proto file location - - Default: Uses Go package paths - -## Troubleshooting - -### Plugin Not Found - -If you get an error that the plugin is not found: - -- Ensure `protoc-gen-slimrpc-python` or `protoc-gen-slimrpc-go` is in your PATH -- Or specify the full path: - - For Python: `--plugin=protoc-gen-slimrpc=/full/path/to/protoc-gen-slimrpc-python` - - For Go: `--plugin=protoc-gen-slimrpc=/full/path/to/protoc-gen-slimrpc-go` - -### Import Errors - -If you encounter import errors: - -- **Python**: Make sure the generated `*_pb2.py` files are in your Python path. Use the `types_import` parameter to customize import paths. -- **Go**: Ensure `go.mod` is properly configured with correct module paths. -- Verify all Protocol Buffer dependencies are generated. From 9b36a0cb73014555d72aab1e572cb14edd17a588 Mon Sep 17 00:00:00 2001 From: Aron Kerekes Date: Tue, 21 Apr 2026 13:33:18 +0200 Subject: [PATCH 10/11] chore: landing page Signed-off-by: Aron Kerekes --- docs/.index | 3 +- docs/index.md | 113 +++++++++++++++++------------------- docs/introduction.md | 60 +++++++++++++++++++ docs/stylesheets/custom.css | 107 ++++++++++++++++++++++++++++++++++ 4 files changed, 222 insertions(+), 61 deletions(-) create mode 100644 docs/introduction.md diff --git a/docs/.index b/docs/.index index 0eeed0a..d63b5db 100644 --- a/docs/.index +++ b/docs/.index @@ -1,5 +1,6 @@ nav: - - Introduction: index.md + - Home: index.md + - Introduction: introduction.md - Getting Started: coffee-agntcy - Observability and Evaluation: obs-and-eval - CSIT: csit diff --git a/docs/index.md b/docs/index.md index 544bbdf..70eac00 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,60 +1,53 @@ -# AGNTCY Origins - -AGNTCY began with Outshift by Cisco's vision of an [Internet of Agents](https://outshift.cisco.com/the-internet-of-agents)—recognizing that AI agents were being built in isolated silos, unable to collaborate across organizational boundaries. Their foundational white paper identified the critical infrastructure gap preventing agents from working together at scale. - -What started as a concept became reality in March 2025 when Outshift launched AGNTCY on GitHub with complete code, specifications, and services, alongside Galileo and LangChain as core maintainers. They built the discovery, identity, messaging, and observability components that agents need to find each other, verify their capabilities, and collaborate securely. - -By July 2025, over 75 companies had joined the effort, leading to AGNTCY's donation to the Linux Foundation with Cisco, Dell Technologies, Google Cloud, Oracle, and Red Hat as formative members. What began as one company's vision became the community-owned infrastructure for the Internet of Agents. - -## Vision - -Agentic AI will accelerate all of human work. Enterprises need to create agentic workflows and applications combining internal and third-party agents to fully leverage the power of AI, accelerate their business, and achieve significant productivity gains. -We believe that an open, interoperable Internet of Agents is the key to enabling the best possible path forward to accelerate innovation and create the most value for all participants, from builders to operators, developers to consumers across all industries and businesses. - -## Mission - -We are an open source collective committed to build the Internet of Agents to be accessible to all. Our mission is to build a diverse, collaborative space to innovate, develop, and maintain software components and services that solve key problems in the domain of agentic workflows and multi-agent applications. - -## Capabilities - -Based on advanced protocols, frameworks, and components, the goal of IoA software infrastructure is to enable and simplify the creation of multi-agent applications through the following steps: - -1. **Discover**: Find and evaluate the best agents for the job. -1. **Compose**: Connect agents into effective workflows across any framework or vendor. -1. **Deploy**: Run multi-agent systems at scale, securely. -1. **Evaluate**: Monitor performance and improve efficiency and efficacy over time. - -## Technical Objectives - -1. **Interoperability**: Establish a common protocol that enables AI agents from different vendors and platforms to communicate and work together efficiently. -2. **Security**: Ensure secure interactions between agents through robust authentication, authorization, and encryption mechanisms. -3. **Scalability**: Design a scalable architecture that leverages the cloud-native stack optimally, supporting a growing number of agents and interactions without compromising performance. -4. **Standardization**: Develop standardized data models and schemas to ensure consistent data representation and validation across the ecosystem. - -## Core Components - -The initial set of IoA components and architecture is outlined below. This is a starting point - as new members join and bring their contributions, the collective will continue to evolve and expand the IoA architecture, components, and interfaces. - -![IoA Stack](assets/ioa_stack.png) - -1. **[Open Agent Schema Framework (OASF)](./oasf/open-agentic-schema-framework.md)**: An OCI based extensible data model allowing to describe agents' attributes and ensuring unique identification of agents. OASF supports the description of A2A agents, MCP servers and can be extended to support other popular formats, such as Copilot agent manifests and many more. Current OASF repo can be found [here](https://github.com/agntcy/oasf), OASF schema documentation can be found [here](https://schema.oasf.outshift.com). -1. **[Agent Directory](./dir/overview.md)**: Allows announcing and discover agents or multi-agent applications which are described using OASF. Any organization can run its directory and keep it in sync with others, forming the Internet of Agents inventory. Agent Directory supports A2A agent cards and MCP server descriptions among other data models. -1. **[Secure Low-Latency Interactive Messaging (SLIM)](./slim/overview.md)**: - * **SLIM** (Secure Low-latency Interactive Messaging): A protocol that defines the standards and guidelines for secure and efficient network-level communication between AI agents. SLIM ensures interoperability and seamless data exchange by specifying message formats, transport mechanisms, and interaction patterns. - * **SLIM Nodes and SDK**: Offers handy secure (MLS and quantum safe) network-level communication services to a group of agents (typically those of a given multi-agent application) through SDK/Libraries. It extends gRPC to support pub/sub interactions in addition to request/reply, streaming, fire & forget and more. -1. **[Identity](./identity/identity.md)**: A system that leverages decentralized technologies to manage and verify the identities of Agents or Tools issued by any organization, ensuring secure and trustworthy interactions. -1. **[Observability and Evaluation](./obs-and-eval/observe-and-eval.md)**: Telemetry collectors, tools and services to enable multi-agent application observability and evaluation. -1. **Security**: Tools and services to trust and protect multi-agent applications. - -[CoffeeAGNTCY](./coffee-agntcy/get-started.md) is a reference implementation demonstrating the core components that can be used to build multi-agent applications. - -The following diagram shows a simplified architecture of the core components described above. - -![IoA Arch](assets/ioa_arch.png) - -## Benefits - -* **Enhanced Collaboration**: By enabling seamless communication and data exchange, IoA fosters collaboration between AI agents, leading to more sophisticated and integrated solutions. -* **Improved Efficiency**: Standardized protocols and frameworks reduce the complexity of integrating diverse AI agents, resulting in faster development and deployment of AI-driven applications. -* **Increased Security**: Robust security mechanisms ensure that interactions between agents are secure, protecting sensitive data, and preventing unauthorized access. -* **Future-Proof Architecture**: The scalable and flexible design of IoA ensures that the ecosystem can grow and adapt to future advancements in AI technology. +--- +hide: + - navigation + - toc + - footer +--- + +# AGNTCY Documentation + +Open documentation for the Internet of Agents: discover agents, describe them with shared schemas, connect them with secure messaging, and verify identity across organizations. + +
+ + Overview + Introduction + Vision, mission, technical objectives, and how the core components fit together. + + + Tutorial + Getting Started + CoffeeAGNTCY reference app—SDKs, patterns, and hands-on flows across AGNTCY components. + + + Discovery + Directory + Announce, discover, and sync agent and MCP listings across the IoA inventory. + + + Schemas + OASF + Browse and validate the Open Agent Schema Framework—classes, skills, and extensions. + + + Messaging + SLIM + Secure Low-Latency Interactive Messaging—protocol, transports, and SDK usage. + + + Trust + Identity + Identity specification—DIDs, verifiable credentials, and agent badges in the IoA. + + + Quality + CSIT + Continuous System Integration Testing—benchmarks, pipelines, and validation across projects. + + + Operations + Observability & Evaluation + Telemetry, tracing, and evaluation for multi-agent workflows and system insight. + +
diff --git a/docs/introduction.md b/docs/introduction.md new file mode 100644 index 0000000..544bbdf --- /dev/null +++ b/docs/introduction.md @@ -0,0 +1,60 @@ +# AGNTCY Origins + +AGNTCY began with Outshift by Cisco's vision of an [Internet of Agents](https://outshift.cisco.com/the-internet-of-agents)—recognizing that AI agents were being built in isolated silos, unable to collaborate across organizational boundaries. Their foundational white paper identified the critical infrastructure gap preventing agents from working together at scale. + +What started as a concept became reality in March 2025 when Outshift launched AGNTCY on GitHub with complete code, specifications, and services, alongside Galileo and LangChain as core maintainers. They built the discovery, identity, messaging, and observability components that agents need to find each other, verify their capabilities, and collaborate securely. + +By July 2025, over 75 companies had joined the effort, leading to AGNTCY's donation to the Linux Foundation with Cisco, Dell Technologies, Google Cloud, Oracle, and Red Hat as formative members. What began as one company's vision became the community-owned infrastructure for the Internet of Agents. + +## Vision + +Agentic AI will accelerate all of human work. Enterprises need to create agentic workflows and applications combining internal and third-party agents to fully leverage the power of AI, accelerate their business, and achieve significant productivity gains. +We believe that an open, interoperable Internet of Agents is the key to enabling the best possible path forward to accelerate innovation and create the most value for all participants, from builders to operators, developers to consumers across all industries and businesses. + +## Mission + +We are an open source collective committed to build the Internet of Agents to be accessible to all. Our mission is to build a diverse, collaborative space to innovate, develop, and maintain software components and services that solve key problems in the domain of agentic workflows and multi-agent applications. + +## Capabilities + +Based on advanced protocols, frameworks, and components, the goal of IoA software infrastructure is to enable and simplify the creation of multi-agent applications through the following steps: + +1. **Discover**: Find and evaluate the best agents for the job. +1. **Compose**: Connect agents into effective workflows across any framework or vendor. +1. **Deploy**: Run multi-agent systems at scale, securely. +1. **Evaluate**: Monitor performance and improve efficiency and efficacy over time. + +## Technical Objectives + +1. **Interoperability**: Establish a common protocol that enables AI agents from different vendors and platforms to communicate and work together efficiently. +2. **Security**: Ensure secure interactions between agents through robust authentication, authorization, and encryption mechanisms. +3. **Scalability**: Design a scalable architecture that leverages the cloud-native stack optimally, supporting a growing number of agents and interactions without compromising performance. +4. **Standardization**: Develop standardized data models and schemas to ensure consistent data representation and validation across the ecosystem. + +## Core Components + +The initial set of IoA components and architecture is outlined below. This is a starting point - as new members join and bring their contributions, the collective will continue to evolve and expand the IoA architecture, components, and interfaces. + +![IoA Stack](assets/ioa_stack.png) + +1. **[Open Agent Schema Framework (OASF)](./oasf/open-agentic-schema-framework.md)**: An OCI based extensible data model allowing to describe agents' attributes and ensuring unique identification of agents. OASF supports the description of A2A agents, MCP servers and can be extended to support other popular formats, such as Copilot agent manifests and many more. Current OASF repo can be found [here](https://github.com/agntcy/oasf), OASF schema documentation can be found [here](https://schema.oasf.outshift.com). +1. **[Agent Directory](./dir/overview.md)**: Allows announcing and discover agents or multi-agent applications which are described using OASF. Any organization can run its directory and keep it in sync with others, forming the Internet of Agents inventory. Agent Directory supports A2A agent cards and MCP server descriptions among other data models. +1. **[Secure Low-Latency Interactive Messaging (SLIM)](./slim/overview.md)**: + * **SLIM** (Secure Low-latency Interactive Messaging): A protocol that defines the standards and guidelines for secure and efficient network-level communication between AI agents. SLIM ensures interoperability and seamless data exchange by specifying message formats, transport mechanisms, and interaction patterns. + * **SLIM Nodes and SDK**: Offers handy secure (MLS and quantum safe) network-level communication services to a group of agents (typically those of a given multi-agent application) through SDK/Libraries. It extends gRPC to support pub/sub interactions in addition to request/reply, streaming, fire & forget and more. +1. **[Identity](./identity/identity.md)**: A system that leverages decentralized technologies to manage and verify the identities of Agents or Tools issued by any organization, ensuring secure and trustworthy interactions. +1. **[Observability and Evaluation](./obs-and-eval/observe-and-eval.md)**: Telemetry collectors, tools and services to enable multi-agent application observability and evaluation. +1. **Security**: Tools and services to trust and protect multi-agent applications. + +[CoffeeAGNTCY](./coffee-agntcy/get-started.md) is a reference implementation demonstrating the core components that can be used to build multi-agent applications. + +The following diagram shows a simplified architecture of the core components described above. + +![IoA Arch](assets/ioa_arch.png) + +## Benefits + +* **Enhanced Collaboration**: By enabling seamless communication and data exchange, IoA fosters collaboration between AI agents, leading to more sophisticated and integrated solutions. +* **Improved Efficiency**: Standardized protocols and frameworks reduce the complexity of integrating diverse AI agents, resulting in faster development and deployment of AI-driven applications. +* **Increased Security**: Robust security mechanisms ensure that interactions between agents are secure, protecting sensitive data, and preventing unauthorized access. +* **Future-Proof Architecture**: The scalable and flexible design of IoA ensures that the ecosystem can grow and adapt to future advancements in AI technology. diff --git a/docs/stylesheets/custom.css b/docs/stylesheets/custom.css index 9f1cbfd..45019bb 100644 --- a/docs/stylesheets/custom.css +++ b/docs/stylesheets/custom.css @@ -178,4 +178,111 @@ html [data-md-color-primary=black] .md-nav--primary .md-nav__title[for=__drawer] [data-md-color-scheme="slate"] .doc-var-tag { border-color: var(--md-default-fg-color--lightest); background-color: var(--md-code-bg-color); +} + +/* Landing hero cards (docs home) */ +.docs-hero-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(17rem, 1fr)); + gap: 1rem; + margin: 1.5rem 0 2rem; +} + +.docs-hero-card { + display: flex; + flex-direction: column; + gap: 0.35rem; + padding: 1.1rem 1.15rem; + border-radius: 0.35rem; + text-decoration: none !important; + color: inherit !important; + border: 1px solid var(--md-default-fg-color--lighter); + box-shadow: 0 0.05rem 0.1rem #0000000d; + transition: + border-color 0.18s ease, + box-shadow 0.18s ease, + transform 0.14s ease, + background 0.22s ease; +} + +/* Light mode: soft, airy gradient (see palette `scheme: default`) */ +[data-md-color-scheme="default"] .docs-hero-card { + background: linear-gradient( + 160deg, + rgb(255 255 255) 0%, + rgb(248 250 254) 38%, + rgb(241 245 252) 72%, + rgb(236 242 250) 100% + ); + border-color: rgba(2, 81, 175, 0.12); +} + +[data-md-color-scheme="default"] .docs-hero-card:hover { + background: linear-gradient( + 160deg, + rgb(255 255 255) 0%, + rgb(250 252 255) 40%, + rgb(244 248 253) 100% + ); +} + +/* Dark mode: lighter stops than before so cards don’t read as heavy slabs */ +[data-md-color-scheme="slate"] .docs-hero-card { + border-color: var(--md-default-fg-color--lightest); + background: linear-gradient( + 160deg, + rgb(18 62 98) 0%, + rgb(10 44 78) 48%, + rgb(6 32 58) 100% + ); +} + +[data-md-color-scheme="slate"] .docs-hero-card:hover { + background: linear-gradient( + 160deg, + rgb(24 72 112) 0%, + rgb(14 52 88) 48%, + rgb(8 38 68) 100% + ); +} + +.docs-hero-card:hover { + border-color: var(--md-accent-fg-color); + box-shadow: 0 0.15rem 0.35rem #00000014; + transform: translateY(-2px); +} + +.docs-hero-card[target="_blank"] .docs-hero-card__title::after { + content: "↗"; + display: inline-block; + margin-left: 0.3em; + font-size: 0.75em; + font-weight: 600; + opacity: 0.75; +} + +.docs-hero-card__label { + font-size: 0.7rem; + font-weight: 700; + letter-spacing: 0.06em; + text-transform: uppercase; + color: var(--md-typeset-a-color); + opacity: 0.95; +} + +.docs-hero-card__title { + font-size: 1.15rem; + font-weight: 700; + line-height: 1.25; +} + +.docs-hero-card__desc { + font-size: 0.82rem; + line-height: 1.45; + color: var(--md-default-fg-color--light); + margin: 0; +} + +[data-md-color-scheme="slate"] .docs-hero-card__desc { + color: var(--md-default-fg-color--lighter); } \ No newline at end of file From d7e96a1d26296a26a1ac6e807c3a347c44d031a9 Mon Sep 17 00:00:00 2001 From: Aron Kerekes Date: Tue, 21 Apr 2026 13:46:54 +0200 Subject: [PATCH 11/11] chore: further edits Signed-off-by: Aron Kerekes --- docs/index.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/index.md b/docs/index.md index 70eac00..5c86a50 100644 --- a/docs/index.md +++ b/docs/index.md @@ -18,7 +18,7 @@ Open documentation for the Internet of Agents: discover agents, describe them wi Tutorial Getting Started - CoffeeAGNTCY reference app—SDKs, patterns, and hands-on flows across AGNTCY components. + CoffeeAGNTCY reference app using SDKs, patterns, and hands-on flows across AGNTCY components. Discovery @@ -28,22 +28,22 @@ Open documentation for the Internet of Agents: discover agents, describe them wi Schemas OASF - Browse and validate the Open Agent Schema Framework—classes, skills, and extensions. + Browse and validate the Open Agent Schema Framework: classes, skills, and extensions. Messaging SLIM - Secure Low-Latency Interactive Messaging—protocol, transports, and SDK usage. + Secure Low-Latency Interactive Messaging: protocol, transports, and SDK usage. Trust Identity - Identity specification—DIDs, verifiable credentials, and agent badges in the IoA. + Identity specification: DIDs, verifiable credentials, and agent badges in the IoA. Quality CSIT - Continuous System Integration Testing—benchmarks, pipelines, and validation across projects. + Continuous System Integration Testing: benchmarks, pipelines, and validation across projects. Operations