From 51f5a9b872983adbe1fc054a3b9f90049c864dd3 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Mon, 15 Dec 2025 19:42:03 +0800 Subject: [PATCH 01/38] support seekdb-python package --- .github/workflows/release-seekdb-python.yml | 64 +++ seekdb-python/README.md | 384 +++++++++++++ seekdb-python/__init__.py | 46 ++ seekdb-python/cli.py | 578 ++++++++++++++++++++ seekdb-python/seekdb_test.py | 49 ++ seekdb-python/setup.py | 238 ++++++++ 6 files changed, 1359 insertions(+) create mode 100644 .github/workflows/release-seekdb-python.yml create mode 100644 seekdb-python/README.md create mode 100644 seekdb-python/__init__.py create mode 100644 seekdb-python/cli.py create mode 100644 seekdb-python/seekdb_test.py create mode 100644 seekdb-python/setup.py diff --git a/.github/workflows/release-seekdb-python.yml b/.github/workflows/release-seekdb-python.yml new file mode 100644 index 0000000..276f9b4 --- /dev/null +++ b/.github/workflows/release-seekdb-python.yml @@ -0,0 +1,64 @@ +name: release seekdb-python + +on: + push: + tags: + - "seekdb-python-v*-*" + +env: + tagName: ${{ github.ref_name }} + +jobs: + build-wheels: + strategy: + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] + runs-on: ${{ matrix.os }} + outputs: + seekdb_version: ${{ steps.extract_version.outputs.seekdb_version }} + package_version: ${{ steps.extract_version.outputs.package_version }} + steps: + - name: Extract version from tag + id: extract_version + run: | + # Extract versions from tag (e.g., 'seekdb-python-v1.0.0-1.0.0.post1') + # Format: seekdb-python-{seekdb_version}-{pylibseekdb_version} + TAG_WITHOUT_PREFIX=$(echo "${{ env.tagName }}" | sed 's/seekdb-python-//') + + # Split by last dash to separate seekdb version and pylibseekdb version + # seekdb_version is everything before the last dash (e.g., 'v1.0.0') + # package_version is everything after the last dash (e.g., '1.0.0.post1') + SEEKDB_VERSION=$(echo "${TAG_WITHOUT_PREFIX}" | sed 's/-[^-]*$//') + PACKAGE_VERSION=$(echo "${TAG_WITHOUT_PREFIX}" | sed 's/^.*-//') + + echo "seekdb_version=${SEEKDB_VERSION}" >> $GITHUB_OUTPUT + echo "package_version=${PACKAGE_VERSION}" >> $GITHUB_OUTPUT + echo "Extracted seekdb version: ${SEEKDB_VERSION}" + echo "Extracted package version: ${PACKAGE_VERSION}" + + - name: Checkout repository + uses: actions/checkout@v5 + + - uses: actions/setup-python@v5 + + - name: Build wheels on ${{ matrix.os }} + uses: pypa/cibuildwheel@v3.3.0 + env: + CIBW_BEFORE_BUILD: "pip install setuptools wheel build" + CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*" + CIBW_SKIP: "*-musllinux_* *-win_* *-macosx_*" + CIBW_ENVIRONMENT: "SEEKDB_GIT_TAG=${{ steps.extract_version.outputs.seekdb_version }} PACKAGE_VERSION=${{ steps.extract_version.outputs.package_version }} BUILD_TYPE=release REBUILD=1 SEEKDB_BUILD_LIBRARY=1" + CIBW_BEFORE_BUILD_LINUX: | + yum install -y -q git wget rpm* cpio make glibc-devel glibc-headers binutils m4 libtool libaio ccache + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28_x86_64 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28_aarch64 + with: + package-dir: seekdb-python + output-dir: wheelhouse + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels + path: wheelhouse/*.whl + retention-days: 30 diff --git a/seekdb-python/README.md b/seekdb-python/README.md new file mode 100644 index 0000000..6212aba --- /dev/null +++ b/seekdb-python/README.md @@ -0,0 +1,384 @@ +## 🚀 What is OceanBase seekdb? + +**OceanBase seekdb** is an AI-native search database that unifies relational, vector, text, JSON and GIS in a single engine, enabling hybrid search and in-database AI workflows. + +--- + +## 🔥 Why OceanBase seekdb? + +| **Feature** | **seekdb** | **OceanBase** | **Chroma** | **Milvus** | **MySQL 9.0** | **PostgreSQL
+pgvector** | **DuckDB** | **Elasticsearch** | +| ------------------------ |:--------------------:|:-------------:|:----------:|:----------:|:-----------------------:|:----------------------------:|:----------:|:-----------------------------------:| +| **Embedded** | ✅ | ❌ | ✅ | ✅ | ❌[1] | ❌ | ✅ | ❌ | +| **Single-Node** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| **Distributed** | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | +| **MySQL Compatible** | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | +| **Vector Search** | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | +| **Full-Text Search** | ✅ | ✅ | ✅ | ⚠️ | ✅ | ✅ | ✅ | ✅ | +| **Hybrid Search** | ✅ | ✅ | ✅ | ✅ | ❌ | ⚠️ | ❌ | ✅ | +| **OLTP** | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | +| **OLAP** | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ⚠️ | +| **License** | Apache 2.0 | MulanPubL 2.0 | Apache 2.0 | Apache 2.0 | GPL 2.0 | PostgreSQL License | MIT | AGPLv3
+SSPLv1
+Elastic 2.0 | +> [1] Embedded capability is removed in MySQL 8.0 +> - ✅ Supported +> - ❌ Not Supported +> - ⚠️ Limited + +## ✨ Key Features + +### Build fast + Hybrid search + Multi model +1. **Build fast:** From prototype to production in minutes: create AI apps using Python, run VectorDBBench on 1C2G. +2. **Hybrid Search:** Combine vector search, full-text search and relational query in a single statement. +3. **Multi-Model:** Support relational, vector, text, JSON and GIS in a single engine. + + +### AI inside + SQL inside +1. **AI Inside:** Run embedding, reranking, LLM inference and prompt management inside the database, supporting a complete document-in/data-out RAG workflow. +2. **SQL Inside:** Powered by the proven OceanBase engine, delivering real-time writes and queries with full ACID compliance, and seamless MySQL ecosystem compatibility. + +## Installation + +```bash +pip install pylibseekdb +``` + +## Requirements + +- CPython 3.8+ +- Linux x86_64, aarch64/arm64 with glibc version >= 2.28 (Alpine Linux is not supported yet) + +--- + +## 🎬 Quick Start + +### Installation + +🐍 Python (Recommended for AI/ML) + +```bash +pip install -U pyseekdb +``` + +### 🎯 AI Search Example + +Build a semantic search system in 5 minutes: + +🗄️ 🐍 Python SDK + +```bash +# install sdk first +pip install -U pyseekdb +``` + +```python +""" +this example demonstrates the most common operations with embedding functions: +1. Create a client connection +2. Create a collection with embedding function +3. Add data using documents (embeddings auto-generated) +4. Query using query texts (embeddings auto-generated) +5. Print query results + +This is a minimal example to get you started quickly with embedding functions. +""" + +import pyseekdb +from pyseekdb import DefaultEmbeddingFunction + +# ==================== Step 1: Create Client Connection ==================== +# You can use embedded mode, server mode, or OceanBase mode +# For this example, we'll use server mode (you can change to embedded or OceanBase) + +# Embedded mode (local SeekDB) +client = pyseekdb.Client( + path="./seekdb.db", + database="test" +) +# Alternative: Server mode (connecting to remote SeekDB server) +# client = pyseekdb.Client( +# host="127.0.0.1", +# port=2881, +# database="test", +# user="root", +# password="" +# ) + +# Alternative: Remote server mode (OceanBase Server) +# client = pyseekdb.Client( +# host="127.0.0.1", +# port=2881, +# tenant="test", # OceanBase default tenant +# database="test", +# user="root", +# password="" +# ) + +# ==================== Step 2: Create a Collection with Embedding Function ==================== +# A collection is like a table that stores documents with vector embeddings +collection_name = "my_simple_collection" + +# Create collection with default embedding function +# The embedding function will automatically convert documents to embeddings +collection = client.create_collection( + name=collection_name, + #embedding_function=DefaultEmbeddingFunction() # Uses default model (384 dimensions) +) + +print(f"Created collection '{collection_name}' with dimension: {collection.dimension}") +print(f"Embedding function: {collection.embedding_function}") + +# ==================== Step 3: Add Data to Collection ==================== +# With embedding function, you can add documents directly without providing embeddings +# The embedding function will automatically generate embeddings from documents + +documents = [ + "Machine learning is a subset of artificial intelligence", + "Python is a popular programming language", + "Vector databases enable semantic search", + "Neural networks are inspired by the human brain", + "Natural language processing helps computers understand text" +] + +ids = ["id1", "id2", "id3", "id4", "id5"] + +# Add data with documents only - embeddings will be auto-generated by embedding function +collection.add( + ids=ids, + documents=documents, # embeddings will be automatically generated + metadatas=[ + {"category": "AI", "index": 0}, + {"category": "Programming", "index": 1}, + {"category": "Database", "index": 2}, + {"category": "AI", "index": 3}, + {"category": "NLP", "index": 4} + ] +) + +print(f"\nAdded {len(documents)} documents to collection") +print("Note: Embeddings were automatically generated from documents using the embedding function") + +# ==================== Step 4: Query the Collection ==================== +# With embedding function, you can query using text directly +# The embedding function will automatically convert query text to query vector + +# Query using text - query vector will be auto-generated by embedding function +query_text = "artificial intelligence and machine learning" + +results = collection.query( + query_texts=query_text, # Query text - will be embedded automatically + n_results=3 # Return top 3 most similar documents +) + +print(f"\nQuery: '{query_text}'") +print(f"Query results: {len(results['ids'][0])} items found") + +# ==================== Step 5: Print Query Results ==================== +for i in range(len(results['ids'][0])): + print(f"\nResult {i+1}:") + print(f" ID: {results['ids'][0][i]}") + print(f" Distance: {results['distances'][0][i]:.4f}") + if results.get('documents'): + print(f" Document: {results['documents'][0][i]}") + if results.get('metadatas'): + print(f" Metadata: {results['metadatas'][0][i]}") + +# ==================== Step 6: Cleanup ==================== +# Delete the collection +client.delete_collection(collection_name) +print(f"\nDeleted collection '{collection_name}'") + +``` +Please refer to the [User Guide](https://github.com/oceanbase/pyseekdb) for more details. + +🗄️ SQL + +```sql +-- Create table with vector column +CREATE TABLE articles ( + id INT PRIMARY KEY, + title TEXT, + content TEXT, + embedding VECTOR(384), + FULLTEXT INDEX idx_fts(content) WITH PARSER ik, + VECTOR INDEX idx_vec (embedding) WITH(DISTANCE=l2, TYPE=hnsw, LIB=vsag) + ) ORGANIZATION = HEAP; + +-- Insert documents with embeddings +-- Note: Embeddings should be pre-computed using your embedding model +INSERT INTO articles (id, title, content, embedding) +VALUES + (1, 'AI and Machine Learning', 'Artificial intelligence is transforming...', '[0.1, 0.2, ...]'), + (2, 'Database Systems', 'Modern databases provide high performance...', '[0.3, 0.4, ...]'), + (3, 'Vector Search', 'Vector databases enable semantic search...', '[0.5, 0.6, ...]'); + +-- Example: Hybrid search combining vector and full-text +-- Replace '[query_embedding]' with your actual query embedding vector +SELECT + title, + content, + l2_distance(embedding, '[query_embedding]') AS vector_distance, + MATCH(content) AGAINST('your keywords' IN NATURAL LANGUAGE MODE) AS text_score +FROM articles +WHERE MATCH(content) AGAINST('your keywords' IN NATURAL LANGUAGE MODE) +ORDER BY vector_distance APPROXIMATE +LIMIT 10; +``` +We suggest developers use sqlalchemy to access data by SQL for python developers. + + +## 📚 Use Cases + +
+ 📖 RAG & Knowledge Retrieval + +Large language models are limited by their training data. RAG introduces timely and trusted external knowledge to improve answer quality and reduce hallucination. seekdb enhances search accuracy through vector search, full-text search, hybrid search, built-in AI functions, and efficient indexing, while multi-level access control safeguards data privacy across heterogeneous knowledge sources. +1. Enterprise QA +2. Customer support +3. Industry insights +4. Personal knowledge + +
+ +
+ 🔍 Semantic Search Engine + +Traditional keyword search struggles to capture intent. Semantic search leverages embeddings and vector search to understand meaning and connect text, images, and other modalities. seekdb's hybrid search and multi-model querying deliver more precise, context-aware results across complex search scenarios. +1. Product search +2. Text-to-image +3. Image-to-product + +
+ +
+ 🎯 Agentic AI Applications + +Agentic AI requires memory, planning, perception, and reasoning. seekdb provides a unified foundation for agents through metadata management, vector/text/mixed queries, multimodal data processing, RAG, built-in AI functions and inference, and robust privacy controls—enabling scalable, production-grade agent systems. +1. Personal assistants +2. Enterprise automation +3. Vertical agents +4. Agent platforms + +
+ +
+ 💻 AI-Assisted Coding & Development + +AI-powered coding combines natural-language understanding and code semantic analysis to enable generation, completion, debugging, testing, and refactoring. seekdb enhances code intelligence with semantic search, multi-model storage for code and documents, isolated multi-project management, and time-travel queries—supporting both local and cloud IDE environments. +1. IDE plugins +2. Design-to-web +3. Local IDEs +4. Web IDEs + +
+ +
+ ⬆️ Enterprise Application Intelligence + +AI transforms enterprise systems from passive tools into proactive collaborators. seekdb provides a unified AI-ready storage layer, fully compatible with MySQL syntax and views, and accelerates mixed workloads with parallel execution and hybrid row-column storage. Legacy applications gain intelligent capabilities with minimal migration across office, workflow, and business analytics scenarios. +1. Document intelligence +2. Business insights +3. Finance systems + +
+ + +
+ 📱 On-Device & Edge AI Applications + +Edge devices—from mobile to vehicle and industrial terminals—operate with constrained compute and storage. seekdb's lightweight architecture supports embedded and micro-server modes, delivering full SQL, JSON, and hybrid search under low resource usage. It integrates seamlessly with OceanBase cloud services to enable unified edge-to-cloud intelligent systems. +1. Personal assistants +2. In-vehicle systems +3. AI education +4. Companion robots +5. Healthcare devices + +
+ +--- + +## 🌟 Ecosystem & Integrations + +
+ +

+ + HuggingFace + + + LangChain + + + LangGraph + + + Dify + + + Coze + + + LlamaIndex + + + Firecrawl + + + FastGPT + + + DB-GPT + + + Camel-AI + + + spring-ai-alibaba + + + Cloudflare Workers AI + + + Jina AI + + + Ragas + + + Instructor + + + Baseten + +

+ +

+Please refer to the [User Guide](docs/user-guide/README.md) for more details. +

+ + +
+ +--- + + +## 🤝 Community & Support + +
+ +

+ + Discord + + + GitHub Discussion + + + Forum + +

+ +
+ + +## License + +This package is licensed under Apache 2.0. diff --git a/seekdb-python/__init__.py b/seekdb-python/__init__.py new file mode 100644 index 0000000..b11ddf9 --- /dev/null +++ b/seekdb-python/__init__.py @@ -0,0 +1,46 @@ +""" +OceanBase seekdb Python Embed + +OceanBase seekdb Python Embed provides Python bindings for OceanBase seekdb, a high-performance embedded database engine. +This package supplies the lightweight interface layer for Python applications, making it easy to interact with seekdb +databases and execute SQL from Python code. +""" + +import sys +import importlib.metadata + +try: + __version__ = importlib.metadata.version("pylibseekdb") +except importlib.metadata.PackageNotFoundError: + __version__ = "0.0.1.dev1" + +__author__ = "OceanBase" + +_LIB_FILE_NAME = "libseekdb_python" + + +def _initialize_module(): + try: + seekdb_module = _load_oblite_module() + attributes = [] + for attr_name in dir(seekdb_module): + if not attr_name.startswith('_'): + setattr(sys.modules[__name__], attr_name, getattr(seekdb_module, attr_name)) + attributes.append(attr_name) + except Exception as e: + print(f"Warning: Failed to import seekdb module: {e}") + attributes = [] + return attributes + +def _load_oblite_module(): + """Load the oblite module""" + + try: + # Import the module + import libseekdb_python + return libseekdb_python + except ImportError as e: + raise ImportError(f"Failed to import {_LIB_FILE_NAME} module: {e}") + +__all__ = ['__version__'] +__all__.extend(_initialize_module()) diff --git a/seekdb-python/cli.py b/seekdb-python/cli.py new file mode 100644 index 0000000..bb6e46b --- /dev/null +++ b/seekdb-python/cli.py @@ -0,0 +1,578 @@ +#!/usr/bin/env python3 +""" +Interactive SQL Client for seekdb + +A command-line tool for executing SQL commands interactively, similar to mysql client. +Works with embedded seekdb databases. + +Usage: + python cli.py [--path PATH] [--database DATABASE] + python cli.py -p ./seekdb.db -d test +""" +import sys +import argparse +import signal +import os +from typing import Optional, List, Any, Tuple + +# Try to import readline for history support (Unix/Linux/Mac) +try: + import readline + HAS_READLINE = True +except ImportError: + # Windows doesn't have readline by default, try pyreadline + try: + import pyreadline3 as readline + HAS_READLINE = True + except ImportError: + HAS_READLINE = False + if sys.platform != 'win32': + print("Warning: readline module not available. History navigation disabled.") + +try: + import tabulate + HAS_TABULATE = True +except ImportError: + HAS_TABULATE = False + +try: + import pylibseekdb as seekdb +except ImportError: + print("Error: seekdb module not found. Please install it first.") + sys.exit(1) + + +def _special_commands_string() -> str: + return """ +Special commands: + \\q, \\quit, \\exit - Exit the client + \\d [table] - Describe table structure + \\dt - List all tables + \\l, \\databases - List all databases + \\c [database] - Connect to a different database + \\h, \\help, \\? - Show this help message + """ +class InteractiveSQLClient: + """Interactive SQL client for seekdb""" + + def __init__(self, path: str = "./seekdb.db", database: str = "test"): + """ + Initialize the interactive SQL client + + Args: + path: Path to seekdb data directory + database: Database name to connect to + """ + self.path = path + self.database = database + self.connection = None + self.cursor: Optional[Any] = None + self.running = True + self.history_file = os.path.expanduser("~/.seekdb_history") + self.sql_history: List[str] = [] + + # Initialize readline if available + if HAS_READLINE: + self._setup_readline() + + # Register signal handlers for graceful exit + signal.signal(signal.SIGINT, self._handle_sigint) + signal.signal(signal.SIGTERM, self._handle_sigterm) + + def _handle_sigint(self, signum, frame): + """Handle Ctrl+C gracefully""" + print("\nUse \\q or \\quit to exit.") + self.running = True # Don't exit on Ctrl+C, just show message + + def _handle_sigterm(self, signum, frame): + """Handle termination signal""" + self.cleanup() + sys.exit(0) + + def _setup_readline(self): + """Setup readline for history and better input handling""" + if not HAS_READLINE: + return + + # Set history file + try: + if os.path.exists(self.history_file): + readline.read_history_file(self.history_file) + except Exception: + pass + + # Configure readline + # Use vi mode if preferred, or keep default emacs mode + # readline.parse_and_bind('set editing-mode vi') # Uncomment for vi mode + + # Set history length + readline.set_history_length(1000) + + # Optional: Set completer for tab completion (can be enhanced later) + # readline.set_completer(self._completer) + # readline.parse_and_bind('tab: complete') + + def _save_history(self): + """Save command history to file""" + if not HAS_READLINE: + return + + try: + readline.write_history_file(self.history_file) + except Exception: + pass + + def connect(self) -> bool: + """Connect to the database""" + try: + print("Openning database...") + seekdb.open(db_dir=self.path) + self.connection = seekdb.connect(database=self.database, autocommit=True) + self.cursor = self.connection.cursor() + # Test connection + self.cursor.execute("SELECT 1") + return True + except Exception as e: + print(f"Error connecting to database: {e}") + return False + + def cleanup(self): + """Close connection and cleanup""" + # Save history before exiting + if HAS_READLINE: + self._save_history() + + if self.cursor: + try: + self.cursor.close() + except Exception: + pass + self.cursor = None + if self.connection: + try: + self.connection.close() + except Exception: + pass + self.connection = None + + def _format_simple_table(self, headers: List[str], rows: List[List[Any]]) -> str: + """ + Format table without tabulate library + + Args: + headers: Column headers + rows: Table rows + + Returns: + Formatted table string + """ + if not headers or not rows: + return "(empty result set)" + + # Calculate column widths + col_widths = [len(str(h)) for h in headers] + for row in rows: + for i, cell in enumerate(row): + if i < len(col_widths): + col_widths[i] = max(col_widths[i], len(str(cell))) + + # Build table + lines = [] + + # Header row + header_line = " | ".join(str(h).ljust(col_widths[i]) for i, h in enumerate(headers)) + lines.append(header_line) + lines.append("-" * len(header_line)) + + # Data rows + for row in rows: + row_line = " | ".join(str(cell).ljust(col_widths[i]) if i < len(col_widths) else str(cell) + for i, cell in enumerate(row)) + lines.append(row_line) + + return "\n".join(lines) + + def _extract_column_names_from_sql(self, sql: str) -> Optional[List[str]]: + """ + Try to extract column names from SQL SELECT statement + + Args: + sql: SQL SELECT statement + + Returns: + List of column names or None if extraction fails + """ + import re + try: + # Match SELECT ... FROM pattern + select_match = re.search(r'SELECT\s+(.+?)\s+FROM', sql, re.IGNORECASE | re.DOTALL) + if not select_match: + return None + + select_clause = select_match.group(1).strip() + + # Handle SELECT * case + if select_clause.strip() == '*': + return None + + # Split by comma, handling nested parentheses + parts = [] + depth = 0 + current = "" + for char in select_clause: + if char == '(': + depth += 1 + elif char == ')': + depth -= 1 + elif char == ',' and depth == 0: + parts.append(current.strip()) + current = "" + continue + current += char + if current: + parts.append(current.strip()) + + # Extract column names + column_names = [] + for part in parts: + part = part.strip() + # Match "AS alias" pattern + as_match = re.search(r'\s+AS\s+["\']?(\w+)["\']?', part, re.IGNORECASE) + if as_match: + column_names.append(as_match.group(1)) + else: + # No alias, extract column name + # Remove backticks and quotes, get last identifier + cleaned = part.replace('`', '').replace('"', '').replace("'", '') + # Get last word (column name) + words = cleaned.split() + if words: + col_name = words[-1] + # Remove function parentheses if present + if '(' in col_name: + col_name = col_name.split('(')[0] + column_names.append(col_name if col_name else f"col_{len(column_names) + 1}") + else: + column_names.append(f"col_{len(column_names) + 1}") + + return column_names if column_names else None + except Exception: + return None + + def format_result(self, result: Any, sql: Optional[str] = None) -> str: + """ + Format query result as a table + + Args: + result: Query result from execute() + sql: Original SQL statement (optional, used to extract column names) + + Returns: + Formatted table string + """ + if not result: + return "(empty result set)" + + # Handle different result formats + if isinstance(result, (list, tuple)): + if len(result) == 0: + return "(empty result set)" + + # Check if result is list of tuples or list of dicts + first_row = result[0] + + if isinstance(first_row, dict): + # Result is list of dictionaries + headers = list(first_row.keys()) + rows = [[row.get(h, '') for h in headers] for row in result] + elif isinstance(first_row, (tuple, list)): + # Result is list of tuples + num_cols = len(first_row) + + # Try to extract column names from SQL + headers = None + if sql: + headers = self._extract_column_names_from_sql(sql) + + # Fallback to generic column names + if not headers or len(headers) != num_cols: + headers = [f"col_{i+1}" for i in range(num_cols)] + + rows = [list(row) for row in result] + else: + # Single value result + return str(result) + + # Format with tabulate if available + if HAS_TABULATE: + try: + return tabulate.tabulate(rows, headers=headers, tablefmt="grid") + except Exception: + pass + + # Fallback to simple table format + return self._format_simple_table(headers, rows) + else: + return str(result) + + def execute_sql(self, sql: str) -> Tuple[bool, str]: + """ + Execute SQL statement + + Args: + sql: SQL statement to execute + + Returns: + Tuple of (success: bool, result_message: str) + """ + if not self.cursor: + return False, "Not connected to database" + + try: + sql_upper = sql.strip().upper() + + # Execute the SQL + self.cursor.execute(sql) + result = self.cursor.fetchall() + + # Format result based on query type + if sql_upper.startswith('SELECT') or sql_upper.startswith('SHOW') or sql_upper.startswith('DESC'): + formatted = self.format_result(result, sql=sql) + return True, formatted + else: + # DML/DDL statements + if hasattr(result, 'rowcount'): + return True, f"Query OK, {result.rowcount} row(s) affected" + else: + return True, "Query OK" + + except Exception as e: + return False, f"Error: {str(e)}" + + def handle_special_command(self, command: str) -> Tuple[bool, Optional[str]]: + """ + Handle special commands (starting with \) + + Args: + command: Special command string + + Returns: + Tuple of (handled: bool, message: Optional[str]) + """ + command = command.strip() + + if not command.startswith('\\'): + return False, None + + parts = command.split(None, 1) + cmd = parts[0].lower() + args = parts[1] if len(parts) > 1 else None + + if cmd in ('\\q', '\\quit', '\\exit'): + self.running = False + return True, "Bye!" + + elif cmd in ('\\h', '\\help', '\\?'): + help_text = _special_commands_string() + """ +SQL commands: + Enter SQL statements terminated by semicolon (;) + Multi-line statements are supported. + +History navigation: + Use Up/Down arrow keys to navigate through command history + History is saved to ~/.seekdb_history + """ + return True, help_text.strip() + + elif cmd == '\\d': + # Describe table + if not args: + return True, "Usage: \\d " + + table_name = args.strip().strip('`').strip('"').strip("'") + success, result = self.execute_sql(f"DESCRIBE `{table_name}`") + return True, result if success else result + + elif cmd == '\\dt': + # List tables + success, result = self.execute_sql("SHOW TABLES") + return True, result if success else result + + elif cmd in ('\\l', '\\databases'): + # List databases + success, result = self.execute_sql( + "SELECT SCHEMA_NAME FROM information_schema.SCHEMATA" + ) + return True, result if success else result + + elif cmd == '\\c': + # Connect to different database + if not args: + return True, f"Current database: {self.database}\nUsage: \\c " + + new_db = args.strip().strip('`').strip('"').strip("'") + try: + self.cleanup() + self.database = new_db + if self.connect(): + return True, f"Connected to database: {new_db}" + else: + return True, f"Failed to connect to database: {new_db}" + except Exception as e: + return True, f"Error: {str(e)}" + + else: + return True, f"Unknown command: {cmd}. Type \\h for help." + + def read_multiline_sql(self) -> Optional[str]: + """ + Read multi-line SQL statement from user input with readline support + + Returns: + Complete SQL statement or None if user wants to quit + """ + lines = [] + prompt = "seekdb[embedded] > " + continuation_prompt = " -> " + + while True: + try: + # input() automatically uses readline if available (supports up/down arrows) + if lines: + line = input(continuation_prompt) + else: + line = input(prompt) + except EOFError: + # User pressed Ctrl+D + print("\nBye!") + return None + except KeyboardInterrupt: + # User pressed Ctrl+C - clear current input + print("\n(Interrupted)") + if lines: + lines = [] + continue + else: + print("Use \\q or \\quit to exit.") + continue + + if not line.strip(): + # Empty line - in multi-line mode, this might be intentional + # In single-line mode, skip it + if not lines: + continue + # In multi-line mode, empty line might be part of the statement + # or user wants to cancel - let's treat it as cancel for now + # User can type semicolon to complete + continue + + # Check for special commands + if line.strip().startswith('\\'): + return line.strip() + + lines.append(line) + + # Check if statement is complete (ends with semicolon) + if line.rstrip().endswith(';'): + sql = '\n'.join(lines) + # Keep the semicolon - most SQL databases accept it + return sql.strip() + + def run(self): + """Run the interactive SQL client""" + print("=" * 60) + print("OceanBase seekdb Embedded Interactive SQL Client") + print("=" * 60) + print(f"Path: {self.path}") + print(f"Database: {self.database}") + print("=" * 60) + print("Type '\\h' for help or '\\q' to quit.") + print() + + if not self.connect(): + print("Failed to connect. Exiting.") + return + + while self.running: + try: + sql = self.read_multiline_sql() + + if sql is None: + break + + if not sql.strip(): + continue + + # Check for special commands + if sql.strip().startswith('\\'): + handled, message = self.handle_special_command(sql) + if message: + print(message) + if not self.running: + break + continue + + # Execute SQL + success, result = self.execute_sql(sql) + print(result) + + # Add to history if successful (for readline) + if success and HAS_READLINE: + # Add the original SQL to readline history + # This allows up/down arrow navigation + try: + readline.add_history(sql) + except Exception: + pass + + if not success: + # Error occurred, but continue running + pass + + print() # Empty line for readability + + except KeyboardInterrupt: + print("\nUse \\q or \\quit to exit.") + continue + except Exception as e: + print(f"Unexpected error: {e}") + continue + + self.cleanup() + print("Connection closed.") + + +def main(): + """Main entry point""" + parser = argparse.ArgumentParser( + description="Interactive SQL client for OceanBase seekdb Embedded", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + %(prog)s + %(prog)s --path ./seekdb.db --database test + %(prog)s -p ./seekdb.db -d mydb + + """ + _special_commands_string() + ) + + parser.add_argument( + '-p', '--path', + default='./seekdb.db', + help='Path to seekdb data directory (default: ./seekdb.db)' + ) + + parser.add_argument( + '-d', '--database', + default='test', + help='Database name (default: test)' + ) + + args = parser.parse_args() + + client = InteractiveSQLClient(path=args.path, database=args.database) + client.run() + + +if __name__ == "__main__": + main() diff --git a/seekdb-python/seekdb_test.py b/seekdb-python/seekdb_test.py new file mode 100644 index 0000000..16981c7 --- /dev/null +++ b/seekdb-python/seekdb_test.py @@ -0,0 +1,49 @@ +#-*- coding: utf-8 -*- +import pylibseekdb as seekdb + +seekdb.open() +conn = seekdb.connect("test") +cursor = conn.cursor() +cursor.execute('drop table if exists doc_table') +cursor.execute('''create table doc_table(c1 int, + vector vector(3), + query varchar(255), + content varchar(255), + vector index idx1(vector) with + (distance=l2, type=hnsw, lib=vsag), + fulltext idx2(query), + fulltext idx3(content))''') + +sql = '''insert into doc_table values + (1, '[1,2,3]', "hello world", "oceanbase Elasticsearch database"), + (2, '[1,2,1]', "hello world, what is your name", "oceanbase mysql database"), + (3, '[1,1,1]', "hello world, how are you", "oceanbase oracle database"), + (4, '[1,3,1]', "real world, where are you from", "postgres oracle database"), + (5, '[1,3,2]', "real world, how old are you", "redis oracle database"), + (6, '[2,1,1]', "hello world, where are you from", "starrocks oceanbase database")''' +cursor.execute(sql) +conn.commit() + +sql = ''' + SET @parm = '{ + "query": { + "bool": { + "should": [ + {"match": {"query": "hi hello"}}, + {"match": { "content": "oceanbase mysql" }} + ] + } + }, + "knn" : { + "field": "vector", + "k": 5, + "query_vector": [1,2,3] + }, + "_source" : ["query", "content", "_keyword_score", "_semantic_score"] + }' + ''' +cursor.execute(sql) +conn.commit() +sql = '''SELECT json_pretty(DBMS_HYBRID_SEARCH.SEARCH('doc_table', @parm))''' +cursor.execute(sql) +print(cursor.fetchall()) diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py new file mode 100644 index 0000000..1c2f3b2 --- /dev/null +++ b/seekdb-python/setup.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +""" +Setup script for seekdb package +""" + +import os +import sys +import subprocess +import shutil +from pathlib import Path +from setuptools import setup, find_packages, Extension +from setuptools.command.build_py import build_py +from setuptools.command.build_ext import build_ext +from wheel.bdist_wheel import bdist_wheel + +# Get the current directory +current_dir = Path(__file__).parent + +def get_version(): + """Get version from the seekdb module""" + return os.environ.get('PACKAGE_VERSION', '0.0.1.dev1') + +def get_description(): + """Get description from the seekdb module""" + return "An AI-Native Search Database. Unifies vector, text, structured and semi-structured data in a single engine, enabling hybrid search and in-database AI workflows." + +def get_long_description(): + """Get long description from README""" + readme_file = current_dir / "README.md" + if readme_file.exists(): + return readme_file.read_text(encoding='utf-8') + return get_description() + +def __library_name(): + return "libseekdb_python" + +def __package_name(): + return "pylibseekdb" + +def get_seekdb_source_dir(): + """Get the seekdb source directory""" + return current_dir / "seekdb" + +def get_project_root(): + """Get the project root directory""" + # Current file is at package/wheel/core/setup.py + # Project root is 3 levels up + root = current_dir.parent.parent.parent + return root.resolve() + +def get_python_version(): + """Get Python version as X.Y""" + return f"{sys.version_info.major}.{sys.version_info.minor}" + +def get_python_home(): + """Get Python home directory""" + python_home = os.environ.get('PYTHON_HOME', '') + if python_home: + return python_home + # Try to infer from sys.executable + return str(Path(sys.executable).parent.parent) + +def clone_repo(source_url: str = None, target_dir: Path = None, git_tag: str = None, delete_if_exists: bool = False): + """ + Clone the seekdb repository + Args: + git_tag: git branch, tag or commit id, if not provided, will clone the latest commit + """ + if source_url is None: + source_url = "https://github.com/oceanbase/seekdb.git" + if target_dir is None: + target_dir = current_dir / "seekdb" + + if target_dir.exists() and (target_dir / ".git").exists(): + if delete_if_exists: + shutil.rmtree(target_dir) + else: + return target_dir + + try: + subprocess.run(f"""mkdir -p {target_dir} \ + && cd ${target_dir} \ + && git init \ + && git remote add origin {source_url} \ + && git fetch --progress --depth=1 origin {git_tag} \ + && git checkout FETCH_HEAD + """, shell=True, check=True, capture_output=True, universal_newlines=True) + except Exception as e: + raise Exception(f"Failed to clone repository: {e}") + return target_dir + +def build_library(): + """Build the libseekdb_python library""" + seekdb_source_dir = get_seekdb_source_dir() + build_type = os.environ.get('BUILD_TYPE', 'release') + python_version = get_python_version() + python_home = get_python_home() + + build_dir = seekdb_source_dir / f"build_{build_type}" + library_path = build_dir / "src" / "observer" / "embed" / f"{__library_name()}.so" + + # Check if library already exists and rebuild flag is not set + rebuild = os.environ.get('REBUILD', '1') + if rebuild == '0' and library_path.exists(): + print(f"Library already exists at {library_path}, skipping build") + return library_path + + print(f"Building library in {seekdb_source_dir}...") + print(f" Build type: {build_type}") + print(f" Python version: {python_version}") + print(f" Python home: {python_home}") + + # Build command + build_cmd = [ + str(seekdb_source_dir / "build.sh"), + build_type, + "--init", + "-DOB_USE_CCACHE=ON", + "-DBUILD_EMBED_MODE=ON", + f"-DPYTHON_VERSION={python_version}", + f"-DCMAKE_PREFIX_PATH={python_home}", + "--make" + ] + + # Run build + result = subprocess.run( + build_cmd, + cwd=str(seekdb_source_dir), + check=True, + capture_output=False + ) + + if not library_path.exists(): + raise FileNotFoundError(f"Build completed but library not found at {library_path}") + + # Strip the shared library to reduce its size, if the strip tool is available + try: + strip_cmd = ["strip", str(library_path)] + subprocess.run(strip_cmd, check=True) + print(f"Stripped library: {library_path}") + except Exception as e: + print(f"Warning: Failed to strip library ({library_path}): {e}") + + lib_size = library_path.stat().st_size + lib_size_mb = lib_size / (1024 * 1024) + print(f"Library size: {lib_size_mb:.2f} MB") + print(f"Library built successfully: {library_path}") + return library_path + +def copy_library(library_path, dest_dir=None): + """Copy the library to the output directory""" + library_name = __library_name() + if dest_dir is None: + dest_dir = current_dir + else: + dest_dir = Path(dest_dir) + + dest_path = dest_dir / f"{library_name}.so" + + print(f"Copying library: {library_path} -> {dest_path}") + dest_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(library_path, dest_path) + print(f" Library copied successfully") + return dest_path + +class BuildExtCommand(build_ext): + """Custom build_ext command that builds the library and treats it as an extension""" + def run(self): + # Clone the repository first + clone_repo(git_tag=os.environ.get('SEEKDB_GIT_TAG', 'main')) + + # Build the library first + if os.environ.get('SEEKDB_BUILD_LIBRARY', '1').upper() in ('1', 'ON', 'YES', 'TRUE'): + library_path = build_library() + + # Copy library to build directory so it's treated as an extension module + build_lib_dir = Path(self.build_lib) / __package_name() + copy_library(library_path, build_lib_dir) + + # Also copy to source directory for package_data + copy_library(library_path, current_dir) + else: + print("Skipping library build (SEEKDB_BUILD_LIBRARY is disabled)") + + # Run the standard build_ext (which will handle the extension modules) + super().run() + +ext_modules = [Extension( + __library_name(), + sources=[], # No sources - we'll copy the pre-built library + extra_objects=[], # Will be handled by build_ext +)] + +setup( + name=__package_name(), + version=get_version(), + description=get_description(), + long_description=get_long_description(), + long_description_content_type="text/markdown", + author="OceanBase", + author_email="open_oceanbase@oceanbase.com", + maintainer="OceanBase", + maintainer_email="open_oceanbase@oceanbase.com", + url="https://github.com/oceanbase/seekdb", + project_urls={ + "Homepage": "https://github.com/oceanbase/seekdb", + "Repository": "https://github.com/oceanbase/seekdb", + "Documentation": "https://github.com/oceanbase/seekdb", + "Bug Tracker": "https://github.com/oceanbase/seekdb/issues", + }, + packages=[__package_name()], + package_dir={f"{__package_name()}": "."}, + include_package_data=True, + ext_modules=ext_modules, + cmdclass={ + 'build_ext': BuildExtCommand + }, + keywords=["database", "oceanbase", "vector-database", "embed", "sql", "AI"], + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: C++", + "Topic :: Database", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + python_requires=">=3.8", + platforms=["manylinux"], + license="Apache 2.0" +) From 13b160211bc26cb70c5a100aecc046c6cd194e3b Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Mon, 15 Dec 2025 19:55:47 +0800 Subject: [PATCH 02/38] valid docker image name (#2) --- .github/workflows/release-seekdb-python.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-seekdb-python.yml b/.github/workflows/release-seekdb-python.yml index 276f9b4..cadc057 100644 --- a/.github/workflows/release-seekdb-python.yml +++ b/.github/workflows/release-seekdb-python.yml @@ -45,13 +45,13 @@ jobs: uses: pypa/cibuildwheel@v3.3.0 env: CIBW_BEFORE_BUILD: "pip install setuptools wheel build" - CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*" + CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" CIBW_SKIP: "*-musllinux_* *-win_* *-macosx_*" CIBW_ENVIRONMENT: "SEEKDB_GIT_TAG=${{ steps.extract_version.outputs.seekdb_version }} PACKAGE_VERSION=${{ steps.extract_version.outputs.package_version }} BUILD_TYPE=release REBUILD=1 SEEKDB_BUILD_LIBRARY=1" CIBW_BEFORE_BUILD_LINUX: | yum install -y -q git wget rpm* cpio make glibc-devel glibc-headers binutils m4 libtool libaio ccache - CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28_x86_64 - CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28_aarch64 + CIBW_MANYLINUX_X86_64_IMAGE: pypa/manylinux_2_28_x86_64 + CIBW_MANYLINUX_AARCH64_IMAGE: pypa/manylinux_2_28_aarch64 with: package-dir: seekdb-python output-dir: wheelhouse From 28bfcce73dd2f700bd0e6e72c3445bb39ddc1816 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Mon, 15 Dec 2025 21:23:53 +0800 Subject: [PATCH 03/38] fix build errors --- .github/workflows/release-seekdb-python.yml | 6 ++-- seekdb-python/setup.py | 37 ++++++++++++++++++--- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release-seekdb-python.yml b/.github/workflows/release-seekdb-python.yml index cadc057..66e6268 100644 --- a/.github/workflows/release-seekdb-python.yml +++ b/.github/workflows/release-seekdb-python.yml @@ -48,10 +48,8 @@ jobs: CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" CIBW_SKIP: "*-musllinux_* *-win_* *-macosx_*" CIBW_ENVIRONMENT: "SEEKDB_GIT_TAG=${{ steps.extract_version.outputs.seekdb_version }} PACKAGE_VERSION=${{ steps.extract_version.outputs.package_version }} BUILD_TYPE=release REBUILD=1 SEEKDB_BUILD_LIBRARY=1" - CIBW_BEFORE_BUILD_LINUX: | - yum install -y -q git wget rpm* cpio make glibc-devel glibc-headers binutils m4 libtool libaio ccache - CIBW_MANYLINUX_X86_64_IMAGE: pypa/manylinux_2_28_x86_64 - CIBW_MANYLINUX_AARCH64_IMAGE: pypa/manylinux_2_28_aarch64 + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 with: package-dir: seekdb-python output-dir: wheelhouse diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index 1c2f3b2..33eed27 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -39,7 +39,7 @@ def __package_name(): def get_seekdb_source_dir(): """Get the seekdb source directory""" - return current_dir / "seekdb" + return current_dir / "seekdb-source" def get_project_root(): """Get the project root directory""" @@ -69,7 +69,7 @@ def clone_repo(source_url: str = None, target_dir: Path = None, git_tag: str = N if source_url is None: source_url = "https://github.com/oceanbase/seekdb.git" if target_dir is None: - target_dir = current_dir / "seekdb" + target_dir = get_seekdb_source_dir() if target_dir.exists() and (target_dir / ".git").exists(): if delete_if_exists: @@ -78,17 +78,38 @@ def clone_repo(source_url: str = None, target_dir: Path = None, git_tag: str = N return target_dir try: + print(f"Cloning repository from {source_url} to {target_dir} with tag {git_tag}") subprocess.run(f"""mkdir -p {target_dir} \ - && cd ${target_dir} \ + && cd {target_dir} \ && git init \ && git remote add origin {source_url} \ && git fetch --progress --depth=1 origin {git_tag} \ && git checkout FETCH_HEAD - """, shell=True, check=True, capture_output=True, universal_newlines=True) + """, + shell=True, + check=True, + capture_output=True, + universal_newlines=True, + text=True) + except subprocess.CalledProcessError as e: + print(e.stdout) + print(e.stderr) + raise Exception(f"Failed to clone repository: {e}") except Exception as e: raise Exception(f"Failed to clone repository: {e}") return target_dir +def install_dependencies() -> None: + """Install dependencies for the library build""" + print("Installing dependencies...") + command = "yum install -y git wget rpm* cpio make glibc-devel glibc-headers binutils m4 libtool libaio ccache" + result = subprocess.run(command, shell=True, check=False, capture_output=True, universal_newlines=True) + if result.returncode != 0: + print(result.stdout) + print(result.stderr) + raise Exception(f"Failed to install dependencies: {result.returncode}") + + def build_library(): """Build the libseekdb_python library""" seekdb_source_dir = get_seekdb_source_dir() @@ -123,12 +144,17 @@ def build_library(): ] # Run build + print(f"Building library with command: {build_cmd}") result = subprocess.run( build_cmd, cwd=str(seekdb_source_dir), check=True, capture_output=False ) + if result.returncode != 0: + print(result.stdout) + print(result.stderr) + raise Exception(f"Failed to build library: {result.returncode}") if not library_path.exists(): raise FileNotFoundError(f"Build completed but library not found at {library_path}") @@ -167,7 +193,8 @@ class BuildExtCommand(build_ext): """Custom build_ext command that builds the library and treats it as an extension""" def run(self): # Clone the repository first - clone_repo(git_tag=os.environ.get('SEEKDB_GIT_TAG', 'main')) + clone_repo(git_tag=os.environ.get('SEEKDB_GIT_TAG', 'master')) + install_dependencies() # Build the library first if os.environ.get('SEEKDB_BUILD_LIBRARY', '1').upper() in ('1', 'ON', 'YES', 'TRUE'): From fb8575e51236a54174d85c39574af03380f6098a Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Tue, 16 Dec 2025 09:33:20 +0800 Subject: [PATCH 04/38] remove rpm* --- seekdb-python/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index 33eed27..0c4723b 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -101,8 +101,9 @@ def clone_repo(source_url: str = None, target_dir: Path = None, git_tag: str = N def install_dependencies() -> None: """Install dependencies for the library build""" + command = "yum install -y git wget cpio make glibc-devel glibc-headers binutils m4 libtool libaio ccache" print("Installing dependencies...") - command = "yum install -y git wget rpm* cpio make glibc-devel glibc-headers binutils m4 libtool libaio ccache" + print(f"command: {command}") result = subprocess.run(command, shell=True, check=False, capture_output=True, universal_newlines=True) if result.returncode != 0: print(result.stdout) From 730038a715c2d283a774cf7584f1a7d40e22f464 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Tue, 16 Dec 2025 09:42:11 +0800 Subject: [PATCH 05/38] fix tag name detect --- .github/workflows/release-seekdb-python.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-seekdb-python.yml b/.github/workflows/release-seekdb-python.yml index 66e6268..1c6b14f 100644 --- a/.github/workflows/release-seekdb-python.yml +++ b/.github/workflows/release-seekdb-python.yml @@ -3,7 +3,7 @@ name: release seekdb-python on: push: tags: - - "seekdb-python-v*-*" + - "seekdb-python-*-*" env: tagName: ${{ github.ref_name }} @@ -21,12 +21,12 @@ jobs: - name: Extract version from tag id: extract_version run: | - # Extract versions from tag (e.g., 'seekdb-python-v1.0.0-1.0.0.post1') + # Extract versions from tag (e.g., 'seekdb-python-1.0.0-1.0.0.post1') # Format: seekdb-python-{seekdb_version}-{pylibseekdb_version} TAG_WITHOUT_PREFIX=$(echo "${{ env.tagName }}" | sed 's/seekdb-python-//') # Split by last dash to separate seekdb version and pylibseekdb version - # seekdb_version is everything before the last dash (e.g., 'v1.0.0') + # seekdb_version is everything before the last dash (e.g., '1.0.0') # package_version is everything after the last dash (e.g., '1.0.0.post1') SEEKDB_VERSION=$(echo "${TAG_WITHOUT_PREFIX}" | sed 's/-[^-]*$//') PACKAGE_VERSION=$(echo "${TAG_WITHOUT_PREFIX}" | sed 's/^.*-//') From af5c0207499c0691e3b0f11f5b5085d6eb63bb9f Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Tue, 16 Dec 2025 11:33:12 +0800 Subject: [PATCH 06/38] install pybind11 --- seekdb-python/setup.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index 0c4723b..cfe52fd 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -101,14 +101,19 @@ def clone_repo(source_url: str = None, target_dir: Path = None, git_tag: str = N def install_dependencies() -> None: """Install dependencies for the library build""" - command = "yum install -y git wget cpio make glibc-devel glibc-headers binutils m4 libtool libaio ccache" + commands = [ + f"{sys.executable} -m ensurepip", + f"{sys.executable} -m pip install build wheel setuptools pybind11 auditwheel", + "yum install -y git wget cpio make glibc-devel glibc-headers binutils m4 libtool libaio ccache" + ] print("Installing dependencies...") - print(f"command: {command}") - result = subprocess.run(command, shell=True, check=False, capture_output=True, universal_newlines=True) - if result.returncode != 0: - print(result.stdout) - print(result.stderr) - raise Exception(f"Failed to install dependencies: {result.returncode}") + print(f"commands: {commands}") + for command in commands: + result = subprocess.run(command, shell=True, check=False, capture_output=True, universal_newlines=True) + if result.returncode != 0: + print(result.stdout) + print(result.stderr) + raise Exception(f"Failed to install dependencies: {result.returncode}") def build_library(): @@ -141,7 +146,8 @@ def build_library(): "-DBUILD_EMBED_MODE=ON", f"-DPYTHON_VERSION={python_version}", f"-DCMAKE_PREFIX_PATH={python_home}", - "--make" + "--make", + "-j3" ] # Run build From 805cc44abeaf2e2ffba312b0d8d585b7a4348994 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Tue, 16 Dec 2025 14:07:14 +0800 Subject: [PATCH 07/38] fix name problem --- seekdb-python/setup.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index cfe52fd..2bcc64c 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -31,10 +31,10 @@ def get_long_description(): return readme_file.read_text(encoding='utf-8') return get_description() -def __library_name(): +def _library_name(): return "libseekdb_python" -def __package_name(): +def _package_name(): return "pylibseekdb" def get_seekdb_source_dir(): @@ -124,7 +124,7 @@ def build_library(): python_home = get_python_home() build_dir = seekdb_source_dir / f"build_{build_type}" - library_path = build_dir / "src" / "observer" / "embed" / f"{__library_name()}.so" + library_path = build_dir / "src" / "observer" / "embed" / f"{_library_name()}.so" # Check if library already exists and rebuild flag is not set rebuild = os.environ.get('REBUILD', '1') @@ -182,7 +182,7 @@ def build_library(): def copy_library(library_path, dest_dir=None): """Copy the library to the output directory""" - library_name = __library_name() + library_name = _library_name() if dest_dir is None: dest_dir = current_dir else: @@ -208,7 +208,7 @@ def run(self): library_path = build_library() # Copy library to build directory so it's treated as an extension module - build_lib_dir = Path(self.build_lib) / __package_name() + build_lib_dir = Path(self.build_lib) / _package_name() copy_library(library_path, build_lib_dir) # Also copy to source directory for package_data @@ -220,13 +220,13 @@ def run(self): super().run() ext_modules = [Extension( - __library_name(), + _library_name(), sources=[], # No sources - we'll copy the pre-built library extra_objects=[], # Will be handled by build_ext )] setup( - name=__package_name(), + name=_package_name(), version=get_version(), description=get_description(), long_description=get_long_description(), @@ -242,8 +242,8 @@ def run(self): "Documentation": "https://github.com/oceanbase/seekdb", "Bug Tracker": "https://github.com/oceanbase/seekdb/issues", }, - packages=[__package_name()], - package_dir={f"{__package_name()}": "."}, + packages=[_package_name()], + package_dir={f"{_package_name()}": "."}, include_package_data=True, ext_modules=ext_modules, cmdclass={ From 36b6173dfa7b8615305a2de4ae5242e15ef89f8e Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Tue, 16 Dec 2025 16:57:54 +0800 Subject: [PATCH 08/38] fix build extensions --- seekdb-python/setup.py | 44 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index 2bcc64c..d68fdf3 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -198,6 +198,44 @@ def copy_library(library_path, dest_dir=None): class BuildExtCommand(build_ext): """Custom build_ext command that builds the library and treats it as an extension""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Store the library path so build_extensions can use it + self.library_path = None + + def build_extensions(self): + """Override to skip compilation and copy pre-built library instead""" + # Skip the standard compilation process since we're using a pre-built library + print("Skipping extension compilation (using pre-built library)") + + # Copy the pre-built library to where setuptools expects the extension + if self.library_path and self.library_path.exists(): + library_name = _library_name() + # Get the extension output path that setuptools expects + ext_fullname = self.get_ext_fullname(library_name) + ext_path = self.get_ext_fullpath(ext_fullname) + ext_path.parent.mkdir(parents=True, exist_ok=True) + + # Copy the pre-built library to the extension output location + shutil.copy2(self.library_path, ext_path) + print(f"Copied pre-built library to extension path: {ext_path}") + print(f" Source: {self.library_path}") + print(f" Destination: {ext_path}") + else: + library_name = _library_name() + # Fallback: try to find library in current_dir + fallback_path = current_dir / f"{library_name}.so" + if fallback_path.exists(): + ext_fullname = self.get_ext_fullname(library_name) + ext_path = self.get_ext_fullpath(ext_fullname) + ext_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(fallback_path, ext_path) + print(f"Copied pre-built library to extension path (from fallback): {ext_path}") + else: + raise FileNotFoundError( + f"Pre-built library not found. Expected at: {self.library_path or fallback_path}" + ) + def run(self): # Clone the repository first clone_repo(git_tag=os.environ.get('SEEKDB_GIT_TAG', 'master')) @@ -207,6 +245,9 @@ def run(self): if os.environ.get('SEEKDB_BUILD_LIBRARY', '1').upper() in ('1', 'ON', 'YES', 'TRUE'): library_path = build_library() + # Store the library path for build_extensions to use + self.library_path = library_path + # Copy library to build directory so it's treated as an extension module build_lib_dir = Path(self.build_lib) / _package_name() copy_library(library_path, build_lib_dir) @@ -216,7 +257,8 @@ def run(self): else: print("Skipping library build (SEEKDB_BUILD_LIBRARY is disabled)") - # Run the standard build_ext (which will handle the extension modules) + # Run the standard build_ext (which will call build_extensions, but we've overridden it) + # This sets up the build directories and calls build_extensions() super().run() ext_modules = [Extension( From 8bc3030c0ebf541948f27eeab8f91563791233a1 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Tue, 16 Dec 2025 20:51:34 +0800 Subject: [PATCH 09/38] free disk space --- .github/workflows/release-seekdb-python.yml | 15 +++++++++++++++ seekdb-python/setup.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-seekdb-python.yml b/.github/workflows/release-seekdb-python.yml index 1c6b14f..c47e154 100644 --- a/.github/workflows/release-seekdb-python.yml +++ b/.github/workflows/release-seekdb-python.yml @@ -36,6 +36,21 @@ jobs: echo "Extracted seekdb version: ${SEEKDB_VERSION}" echo "Extracted package version: ${PACKAGE_VERSION}" + - name: Free Disk Space + uses: insightsengineering/disk-space-reclaimer@v1 + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tools-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + - name: Checkout repository uses: actions/checkout@v5 diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index d68fdf3..edf6f4a 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -147,7 +147,7 @@ def build_library(): f"-DPYTHON_VERSION={python_version}", f"-DCMAKE_PREFIX_PATH={python_home}", "--make", - "-j3" + "-j2" ] # Run build From ea910cc9c3dae968bdd25e3a480b06991cdb083b Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Wed, 17 Dec 2025 10:07:06 +0800 Subject: [PATCH 10/38] fix str.parent --- seekdb-python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index edf6f4a..ee2b815 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -213,7 +213,7 @@ def build_extensions(self): library_name = _library_name() # Get the extension output path that setuptools expects ext_fullname = self.get_ext_fullname(library_name) - ext_path = self.get_ext_fullpath(ext_fullname) + ext_path = Path(self.get_ext_fullpath(ext_fullname)) ext_path.parent.mkdir(parents=True, exist_ok=True) # Copy the pre-built library to the extension output location From b245e7259b236f3ad9057b00c88831fb2a0c2b2a Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Wed, 17 Dec 2025 14:34:49 +0800 Subject: [PATCH 11/38] disk full space --- .github/workflows/release-seekdb-python.yml | 2 +- seekdb-python/setup.py | 18 ++++++------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release-seekdb-python.yml b/.github/workflows/release-seekdb-python.yml index c47e154..56e885f 100644 --- a/.github/workflows/release-seekdb-python.yml +++ b/.github/workflows/release-seekdb-python.yml @@ -60,7 +60,7 @@ jobs: uses: pypa/cibuildwheel@v3.3.0 env: CIBW_BEFORE_BUILD: "pip install setuptools wheel build" - CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" + CIBW_BUILD: "cp38-* cp39-*" CIBW_SKIP: "*-musllinux_* *-win_* *-macosx_*" CIBW_ENVIRONMENT: "SEEKDB_GIT_TAG=${{ steps.extract_version.outputs.seekdb_version }} PACKAGE_VERSION=${{ steps.extract_version.outputs.package_version }} BUILD_TYPE=release REBUILD=1 SEEKDB_BUILD_LIBRARY=1" CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index ee2b815..937045b 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -242,20 +242,14 @@ def run(self): install_dependencies() # Build the library first - if os.environ.get('SEEKDB_BUILD_LIBRARY', '1').upper() in ('1', 'ON', 'YES', 'TRUE'): - library_path = build_library() + library_path = build_library() - # Store the library path for build_extensions to use - self.library_path = library_path + # Store the library path for build_extensions to use + self.library_path = library_path - # Copy library to build directory so it's treated as an extension module - build_lib_dir = Path(self.build_lib) / _package_name() - copy_library(library_path, build_lib_dir) - - # Also copy to source directory for package_data - copy_library(library_path, current_dir) - else: - print("Skipping library build (SEEKDB_BUILD_LIBRARY is disabled)") + # Copy library to build directory so it's treated as an extension module + build_lib_dir = Path(self.build_lib) / _package_name() + copy_library(library_path, build_lib_dir) # Run the standard build_ext (which will call build_extensions, but we've overridden it) # This sets up the build directories and calls build_extensions() From 06532b21cf5737d7a2ba7de96276f22e2964eb8c Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Thu, 18 Dec 2025 11:51:14 +0800 Subject: [PATCH 12/38] fix artifact name and remove one reduntant library --- .github/workflows/release-seekdb-python.yml | 5 +++-- seekdb-python/setup.py | 6 +----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-seekdb-python.yml b/.github/workflows/release-seekdb-python.yml index 56e885f..948e82a 100644 --- a/.github/workflows/release-seekdb-python.yml +++ b/.github/workflows/release-seekdb-python.yml @@ -70,8 +70,9 @@ jobs: output-dir: wheelhouse - name: Upload wheels - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: - name: wheels + name: wheels-${{ steps.extract_version.outputs.package_version }}-${{ runner.arch }} path: wheelhouse/*.whl retention-days: 30 + compression-level: 0 diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index 937045b..aaabc4f 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -247,16 +247,12 @@ def run(self): # Store the library path for build_extensions to use self.library_path = library_path - # Copy library to build directory so it's treated as an extension module - build_lib_dir = Path(self.build_lib) / _package_name() - copy_library(library_path, build_lib_dir) - # Run the standard build_ext (which will call build_extensions, but we've overridden it) # This sets up the build directories and calls build_extensions() super().run() ext_modules = [Extension( - _library_name(), + f"{_package_name()}.{_library_name()}", sources=[], # No sources - we'll copy the pre-built library extra_objects=[], # Will be handled by build_ext )] From 2daedcaf5253bf703f8b7e55c8b0c9c3f2c7cdad Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Thu, 18 Dec 2025 19:31:11 +0800 Subject: [PATCH 13/38] mv so to pylibseekdb directory --- seekdb-python/setup.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index aaabc4f..194a63f 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -208,12 +208,13 @@ def build_extensions(self): # Skip the standard compilation process since we're using a pre-built library print("Skipping extension compilation (using pre-built library)") + # Use the full extension name (e.g., pylibseekdb.libseekdb_python) + full_ext_name = f"{_package_name()}.{_library_name()}" + # Copy the pre-built library to where setuptools expects the extension if self.library_path and self.library_path.exists(): - library_name = _library_name() # Get the extension output path that setuptools expects - ext_fullname = self.get_ext_fullname(library_name) - ext_path = Path(self.get_ext_fullpath(ext_fullname)) + ext_path = Path(self.get_ext_fullpath(full_ext_name)) ext_path.parent.mkdir(parents=True, exist_ok=True) # Copy the pre-built library to the extension output location @@ -222,12 +223,10 @@ def build_extensions(self): print(f" Source: {self.library_path}") print(f" Destination: {ext_path}") else: - library_name = _library_name() # Fallback: try to find library in current_dir - fallback_path = current_dir / f"{library_name}.so" + fallback_path = current_dir / f"{_library_name()}.so" if fallback_path.exists(): - ext_fullname = self.get_ext_fullname(library_name) - ext_path = self.get_ext_fullpath(ext_fullname) + ext_path = Path(self.get_ext_fullpath(full_ext_name)) ext_path.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(fallback_path, ext_path) print(f"Copied pre-built library to extension path (from fallback): {ext_path}") From 2fcf883e14da1f3eda1215921d78049a7e3d2964 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Fri, 19 Dec 2025 10:41:56 +0800 Subject: [PATCH 14/38] add pylibseekdb module path to sys.path --- seekdb-python/__init__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/seekdb-python/__init__.py b/seekdb-python/__init__.py index b11ddf9..4f02eac 100644 --- a/seekdb-python/__init__.py +++ b/seekdb-python/__init__.py @@ -7,10 +7,14 @@ """ import sys +import importlib.util import importlib.metadata +def _package_name(): + return "pylibseekdb" + try: - __version__ = importlib.metadata.version("pylibseekdb") + __version__ = importlib.metadata.version(_package_name()) except importlib.metadata.PackageNotFoundError: __version__ = "0.0.1.dev1" @@ -37,6 +41,13 @@ def _load_oblite_module(): try: # Import the module + # Attempt to find the pylibseekdb module path and add it to sys.path + spec = importlib.util.find_spec(_package_name()) + if spec and spec.submodule_search_locations: + module_path = list(spec.submodule_search_locations)[0] + if module_path not in sys.path: + sys.path.insert(0, module_path) + import libseekdb_python return libseekdb_python except ImportError as e: From 42a07a4b588a2e48027c9e207d1dce268dbd8101 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Fri, 19 Dec 2025 11:20:19 +0800 Subject: [PATCH 15/38] support publish --- .github/workflows/release-seekdb-python.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/release-seekdb-python.yml b/.github/workflows/release-seekdb-python.yml index 948e82a..14b1787 100644 --- a/.github/workflows/release-seekdb-python.yml +++ b/.github/workflows/release-seekdb-python.yml @@ -63,6 +63,7 @@ jobs: CIBW_BUILD: "cp38-* cp39-*" CIBW_SKIP: "*-musllinux_* *-win_* *-macosx_*" CIBW_ENVIRONMENT: "SEEKDB_GIT_TAG=${{ steps.extract_version.outputs.seekdb_version }} PACKAGE_VERSION=${{ steps.extract_version.outputs.package_version }} BUILD_TYPE=release REBUILD=1 SEEKDB_BUILD_LIBRARY=1" + CIBW_TEST_COMMAND: "python seekdb_test.py" CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 with: @@ -76,3 +77,23 @@ jobs: path: wheelhouse/*.whl retention-days: 30 compression-level: 0 + + publish: + needs: build-wheels + runs-on: ubuntu-latest + permissions: + id-token: write # OIDC needs this + contents: read + steps: + - name: Download all wheels + uses: actions/download-artifact@v6 + with: + path: wheelhouse + pattern: wheels-${{ needs.build-wheels.outputs.package_version }}-* + merge-multiple: true + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1.13 + with: + packages-dir: wheelhouse + skip-existing: true From fdb625ea0993a05d284e513a5b011c3918b96113 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Fri, 19 Dec 2025 14:17:35 +0800 Subject: [PATCH 16/38] fix test and build other python version --- .github/workflows/release-seekdb-python.yml | 6 ++---- seekdb-python/setup.py | 21 ++++++++++----------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release-seekdb-python.yml b/.github/workflows/release-seekdb-python.yml index 14b1787..b61652d 100644 --- a/.github/workflows/release-seekdb-python.yml +++ b/.github/workflows/release-seekdb-python.yml @@ -54,16 +54,14 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 - - uses: actions/setup-python@v5 - - name: Build wheels on ${{ matrix.os }} uses: pypa/cibuildwheel@v3.3.0 env: CIBW_BEFORE_BUILD: "pip install setuptools wheel build" - CIBW_BUILD: "cp38-* cp39-*" + CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" CIBW_SKIP: "*-musllinux_* *-win_* *-macosx_*" CIBW_ENVIRONMENT: "SEEKDB_GIT_TAG=${{ steps.extract_version.outputs.seekdb_version }} PACKAGE_VERSION=${{ steps.extract_version.outputs.package_version }} BUILD_TYPE=release REBUILD=1 SEEKDB_BUILD_LIBRARY=1" - CIBW_TEST_COMMAND: "python seekdb_test.py" + CIBW_TEST_COMMAND: "python seekdb-python/seekdb_test.py" CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 with: diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index 194a63f..351e8b6 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -222,18 +222,17 @@ def build_extensions(self): print(f"Copied pre-built library to extension path: {ext_path}") print(f" Source: {self.library_path}") print(f" Destination: {ext_path}") + + # Delete the build_dir + seekdb_source_dir = get_seekdb_source_dir() + build_type = os.environ.get('BUILD_TYPE', 'release') + build_dir = seekdb_source_dir / f"build_{build_type}" + shutil.rmtree(build_dir) + print(f"Deleted build_dir: {build_dir}") else: - # Fallback: try to find library in current_dir - fallback_path = current_dir / f"{_library_name()}.so" - if fallback_path.exists(): - ext_path = Path(self.get_ext_fullpath(full_ext_name)) - ext_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(fallback_path, ext_path) - print(f"Copied pre-built library to extension path (from fallback): {ext_path}") - else: - raise FileNotFoundError( - f"Pre-built library not found. Expected at: {self.library_path or fallback_path}" - ) + raise FileNotFoundError( + f"Pre-built library not found. Expected at: {self.library_path}" + ) def run(self): # Clone the repository first From 59b2fc9ec607400ba52dc964b0f8d6a8ddd0d174 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Fri, 19 Dec 2025 18:09:43 +0800 Subject: [PATCH 17/38] fix cibuildwheel test --- .github/workflows/release-seekdb-python.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-seekdb-python.yml b/.github/workflows/release-seekdb-python.yml index b61652d..07a5c81 100644 --- a/.github/workflows/release-seekdb-python.yml +++ b/.github/workflows/release-seekdb-python.yml @@ -61,11 +61,13 @@ jobs: CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" CIBW_SKIP: "*-musllinux_* *-win_* *-macosx_*" CIBW_ENVIRONMENT: "SEEKDB_GIT_TAG=${{ steps.extract_version.outputs.seekdb_version }} PACKAGE_VERSION=${{ steps.extract_version.outputs.package_version }} BUILD_TYPE=release REBUILD=1 SEEKDB_BUILD_LIBRARY=1" - CIBW_TEST_COMMAND: "python seekdb-python/seekdb_test.py" + CIBW_TEST_COMMAND: "python seekdb_test.py" + CIBW_TEST_REQUIRES: "" CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 with: package-dir: seekdb-python + test-sources: seekdb-python/seekdb_test.py output-dir: wheelhouse - name: Upload wheels From a7c3c44ca7f86bbcde8968a5f4418cdf9665c859 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Mon, 22 Dec 2025 09:33:11 +0800 Subject: [PATCH 18/38] fix test command --- .github/workflows/release-seekdb-python.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release-seekdb-python.yml b/.github/workflows/release-seekdb-python.yml index 07a5c81..0fa6d69 100644 --- a/.github/workflows/release-seekdb-python.yml +++ b/.github/workflows/release-seekdb-python.yml @@ -61,13 +61,12 @@ jobs: CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" CIBW_SKIP: "*-musllinux_* *-win_* *-macosx_*" CIBW_ENVIRONMENT: "SEEKDB_GIT_TAG=${{ steps.extract_version.outputs.seekdb_version }} PACKAGE_VERSION=${{ steps.extract_version.outputs.package_version }} BUILD_TYPE=release REBUILD=1 SEEKDB_BUILD_LIBRARY=1" - CIBW_TEST_COMMAND: "python seekdb_test.py" + CIBW_TEST_COMMAND: "python {project}/seekdb-python/seekdb_test.py" CIBW_TEST_REQUIRES: "" CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 with: package-dir: seekdb-python - test-sources: seekdb-python/seekdb_test.py output-dir: wheelhouse - name: Upload wheels From 418c243df7f831a4b0aacf7b01fd14f641cf4fc3 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Mon, 22 Dec 2025 14:25:17 +0800 Subject: [PATCH 19/38] ccache stat and DEFAULT_LOG_LEVEL=WARN --- seekdb-python/setup.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index 351e8b6..867e512 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -3,6 +3,7 @@ Setup script for seekdb package """ +from filecmp import clear_cache import os import sys import subprocess @@ -137,6 +138,10 @@ def build_library(): print(f" Python version: {python_version}") print(f" Python home: {python_home}") + # clear ccache stat + clear_cache_cmd = "ccache -z" + subprocess.run(clear_cache_cmd, shell=True, check=False, capture_output=True, universal_newlines=True) + # Build command build_cmd = [ str(seekdb_source_dir / "build.sh"), @@ -144,6 +149,7 @@ def build_library(): "--init", "-DOB_USE_CCACHE=ON", "-DBUILD_EMBED_MODE=ON", + "-DEFAULT_LOG_LEVEL=WARN", f"-DPYTHON_VERSION={python_version}", f"-DCMAKE_PREFIX_PATH={python_home}", "--make", @@ -166,6 +172,15 @@ def build_library(): if not library_path.exists(): raise FileNotFoundError(f"Build completed but library not found at {library_path}") + # ccache stat + stat_ccache_cmd = "ccache -s" + result = subprocess.run(stat_ccache_cmd, shell=True, check=False, capture_output=True, universal_newlines=True) + if result.returncode != 0: + print(result.stdout) + print(result.stderr) + raise Exception(f"Failed to get ccache stat: {result.returncode}") + print(f"ccache stat: {result.stdout}") + # Strip the shared library to reduce its size, if the strip tool is available try: strip_cmd = ["strip", str(library_path)] From a87a468920861169763c90badea4314a3488645e Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Mon, 22 Dec 2025 21:27:10 +0800 Subject: [PATCH 20/38] only remove embedded directory --- seekdb-python/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index 867e512..f06e07e 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -149,7 +149,7 @@ def build_library(): "--init", "-DOB_USE_CCACHE=ON", "-DBUILD_EMBED_MODE=ON", - "-DEFAULT_LOG_LEVEL=WARN", + "-DDEFAULT_LOG_LEVEL=WARN", f"-DPYTHON_VERSION={python_version}", f"-DCMAKE_PREFIX_PATH={python_home}", "--make", @@ -241,7 +241,7 @@ def build_extensions(self): # Delete the build_dir seekdb_source_dir = get_seekdb_source_dir() build_type = os.environ.get('BUILD_TYPE', 'release') - build_dir = seekdb_source_dir / f"build_{build_type}" + build_dir = seekdb_source_dir / f"build_{build_type}" / "src" / "observer" / "embed" shutil.rmtree(build_dir) print(f"Deleted build_dir: {build_dir}") else: From 74e085fe78e4a364d686eee6fced5cdf7efd7d41 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Tue, 23 Dec 2025 09:23:28 +0800 Subject: [PATCH 21/38] fix issues from copilot --- seekdb-python/cli.py | 2 +- seekdb-python/setup.py | 13 +++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/seekdb-python/cli.py b/seekdb-python/cli.py index bb6e46b..b4bc724 100644 --- a/seekdb-python/cli.py +++ b/seekdb-python/cli.py @@ -125,7 +125,7 @@ def _save_history(self): def connect(self) -> bool: """Connect to the database""" try: - print("Openning database...") + print("Opening database...") seekdb.open(db_dir=self.path) self.connection = seekdb.connect(database=self.database, autocommit=True) self.cursor = self.connection.cursor() diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index f06e07e..3244187 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -3,16 +3,13 @@ Setup script for seekdb package """ -from filecmp import clear_cache import os import sys import subprocess import shutil from pathlib import Path -from setuptools import setup, find_packages, Extension -from setuptools.command.build_py import build_py +from setuptools import setup, Extension from setuptools.command.build_ext import build_ext -from wheel.bdist_wheel import bdist_wheel # Get the current directory current_dir = Path(__file__).parent @@ -79,12 +76,12 @@ def clone_repo(source_url: str = None, target_dir: Path = None, git_tag: str = N return target_dir try: - print(f"Cloning repository from {source_url} to {target_dir} with tag {git_tag}") + print(f"Cloning repository from {source_url} to '{target_dir}' with tag '{git_tag}'") subprocess.run(f"""mkdir -p {target_dir} \ && cd {target_dir} \ && git init \ && git remote add origin {source_url} \ - && git fetch --progress --depth=1 origin {git_tag} \ + && git fetch --progress --depth=1 origin '{git_tag}' \ && git checkout FETCH_HEAD """, shell=True, @@ -164,10 +161,6 @@ def build_library(): check=True, capture_output=False ) - if result.returncode != 0: - print(result.stdout) - print(result.stderr) - raise Exception(f"Failed to build library: {result.returncode}") if not library_path.exists(): raise FileNotFoundError(f"Build completed but library not found at {library_path}") From 07c11293d3d930e5f0effdfd947adbacfa165da3 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Wed, 24 Dec 2025 11:08:50 +0800 Subject: [PATCH 22/38] fix default_log_level --- seekdb-python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index 3244187..16ed9c4 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -146,7 +146,7 @@ def build_library(): "--init", "-DOB_USE_CCACHE=ON", "-DBUILD_EMBED_MODE=ON", - "-DDEFAULT_LOG_LEVEL=WARN", + "-DDEFAULT_LOG_LEVEL=\"WARN\"", f"-DPYTHON_VERSION={python_version}", f"-DCMAKE_PREFIX_PATH={python_home}", "--make", From 2a6e4a2fb706210d35356326fba77fac588fa639 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Wed, 24 Dec 2025 11:42:54 +0800 Subject: [PATCH 23/38] fix default_log_level --- seekdb-python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seekdb-python/setup.py b/seekdb-python/setup.py index 16ed9c4..910a891 100644 --- a/seekdb-python/setup.py +++ b/seekdb-python/setup.py @@ -146,7 +146,7 @@ def build_library(): "--init", "-DOB_USE_CCACHE=ON", "-DBUILD_EMBED_MODE=ON", - "-DDEFAULT_LOG_LEVEL=\"WARN\"", + "-DDEFAULT_LOG_LEVEL=OB_LOG_LEVEL_DBA_WARN", f"-DPYTHON_VERSION={python_version}", f"-DCMAKE_PREFIX_PATH={python_home}", "--make", From 1bf9b7bad4d2d6c8a9bdefab1b9f79fed6d1e715 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Thu, 29 Jan 2026 20:09:01 +0800 Subject: [PATCH 24/38] Update Python version requirement to CPython >= 3.11 --- seekdb-python/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/seekdb-python/README.md b/seekdb-python/README.md index 6212aba..3188262 100644 --- a/seekdb-python/README.md +++ b/seekdb-python/README.md @@ -43,8 +43,9 @@ pip install pylibseekdb ## Requirements -- CPython 3.8+ +- CPython >= 3.11 - Linux x86_64, aarch64/arm64 with glibc version >= 2.28 (Alpine Linux is not supported yet) +- MacOS >= 15.6 --- From 31676b3de489c1202661fd9f59fd190ca65a90a0 Mon Sep 17 00:00:00 2001 From: Chris Sun Date: Thu, 25 Dec 2025 15:19:58 +0800 Subject: [PATCH 25/38] support to set parameter with "=" in the value for obproxy-ce (#73) --- obproxy-ce/start.sh | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/obproxy-ce/start.sh b/obproxy-ce/start.sh index feffc6d..6e4fc2d 100755 --- a/obproxy-ce/start.sh +++ b/obproxy-ce/start.sh @@ -1,15 +1,15 @@ #!/bin/bash -if [ -z $APP_NAME ]; then +if [ -z "$APP_NAME" ]; then echo "env variable APP_NAME is required" exit 1 fi -if [ -z $PROXYRO_PASSWORD_HASH ]; then +if [ -z "$PROXYRO_PASSWORD_HASH" ]; then PROXYRO_PASSWORD_HASH=$(echo -n "$PROXYRO_PASSWORD" | sha1sum | awk '{print $1}') fi -if [ -z $PROSYSYS_PASSWORD_HASH ]; then +if [ -z "$PROXYSYS_PASSWORD_HASH" ]; then PROXYSYS_PASSWORD_HASH=$(echo -n "$PROXYSYS_PASSWORD" | sha1sum | awk '{print $1}') fi @@ -17,38 +17,40 @@ opts="obproxy_sys_password=$PROXYSYS_PASSWORD_HASH" function concat_opts { if [ -z "$1" ]; then - echo $2 + echo "$2" elif [ -z "$2" ]; then - echo $1 + echo "$1" else echo "$1,$2" fi } -[ -z "$ODP_PROMETHEUS_SYNC_INTERVAL" ] && opts=$(concat_opts $opts "prometheus_sync_interval=1s") -[ -z "$ODP_ENABLE_METADB_USED" ] && opts=$(concat_opts $opts "enable_metadb_used=false") -[ -z "$ODP_SKIP_PROXY_SYS_PRIVATE_CHECK" ] && opts=$(concat_opts $opts "skip_proxy_sys_private_check=true") -[ -z "$ODP_LOG_DIR_SIZE_THRESHOLD" ] && opts=$(concat_opts $opts "log_dir_size_threshold=10G") -[ -z "$ODP_ENABLE_PROXY_SCRAMBLE" ] && opts=$(concat_opts $opts "enable_proxy_scramble=true") -[ -z "$ODP_ENABLE_STRICT_KERNEL_RELEASE" ] && opts=$(concat_opts $opts "enable_strict_kernel_release=false") - -while IFS='=' read -r key value; do - # If the key has prefix "ODP_" then add it to the opts - if [[ $key == ODP_* ]]; then +[ -z "$ODP_PROMETHEUS_SYNC_INTERVAL" ] && opts=$(concat_opts "$opts" "prometheus_sync_interval=1s") +[ -z "$ODP_ENABLE_METADB_USED" ] && opts=$(concat_opts "$opts" "enable_metadb_used=false") +[ -z "$ODP_SKIP_PROXY_SYS_PRIVATE_CHECK" ] && opts=$(concat_opts "$opts" "skip_proxy_sys_private_check=true") +[ -z "$ODP_LOG_DIR_SIZE_THRESHOLD" ] && opts=$(concat_opts "$opts" "log_dir_size_threshold=10G") +[ -z "$ODP_ENABLE_PROXY_SCRAMBLE" ] && opts=$(concat_opts "$opts" "enable_proxy_scramble=true") +[ -z "$ODP_ENABLE_STRICT_KERNEL_RELEASE" ] && opts=$(concat_opts "$opts" "enable_strict_kernel_release=false") + +while IFS= read -r line; do + # If the line has prefix "ODP_" then add it to the opts + if [[ $line == ODP_* ]]; then + key="${line%%=*}" + value="${line#*=}" # Remove the prefix "ODP_" from the key and transform to lower case - key=$(echo $key | sed 's/^ODP_//g' | tr '[:upper:]' '[:lower:]') - opts=$(concat_opts $opts "$(printf "%s=%s" "$key" "$value")") + key=$(echo "${key#ODP_}" | tr '[:upper:]' '[:lower:]') + opts=$(concat_opts "$opts" "$key=$value") fi done < <(env) echo "$opts" -if [ ! -z $CONFIG_URL ]; then +if [ ! -z "$CONFIG_URL" ]; then echo "use config server" - cd /home/admin/obproxy && /home/admin/obproxy/bin/obproxy -p 2883 -l 2884 -s 2885 -n ${APP_NAME} -o observer_sys_password=${PROXYRO_PASSWORD_HASH},obproxy_config_server_url="${CONFIG_URL}",$opts --nodaemon -elif [ ! -z $RS_LIST ]; then + cd /home/admin/obproxy && /home/admin/obproxy/bin/obproxy -p 2883 -l 2884 -s 2885 -n "${APP_NAME}" -o "observer_sys_password=${PROXYRO_PASSWORD_HASH},obproxy_config_server_url=${CONFIG_URL},${opts}" --nodaemon +elif [ ! -z "$RS_LIST" ]; then echo "use rslist" - cd /home/admin/obproxy && /home/admin/obproxy/bin/obproxy -p 2883 -l 2884 -s 2885 -n ${APP_NAME} -c ${OB_CLUSTER} -r "${RS_LIST}" -o observer_sys_password=${PROXYRO_PASSWORD_HASH},$opts --nodaemon + cd /home/admin/obproxy && /home/admin/obproxy/bin/obproxy -p 2883 -l 2884 -s 2885 -n "${APP_NAME}" -c "${OB_CLUSTER}" -r "${RS_LIST}" -o "observer_sys_password=${PROXYRO_PASSWORD_HASH},${opts}" --nodaemon else echo "no config server or rs list" exit 1 From de234cfdab128974143374e3858437650f69f9a7 Mon Sep 17 00:00:00 2001 From: Chris Sun Date: Thu, 25 Dec 2025 15:23:55 +0800 Subject: [PATCH 26/38] Add helm chart for seekdb (#74) * support to set parameter with "=" in the value for obproxy-ce * add helm chart for seekdb * adjust probe check delay and interval * add githubaction for release helm chart * change default resource setting * fix template file * bump version to 0.1.1 * trigger action on branch *-chart --- .github/workflows/release-charts.yml | 38 ++++++ charts/seekdb/Chart.yaml | 10 ++ charts/seekdb/templates/NOTES.txt | 30 +++++ charts/seekdb/templates/_helpers.tpl | 62 ++++++++++ charts/seekdb/templates/secret.yaml | 20 ++++ charts/seekdb/templates/service-obshell.yaml | 24 ++++ charts/seekdb/templates/service-sql.yaml | 24 ++++ charts/seekdb/templates/serviceaccount.yaml | 12 ++ charts/seekdb/templates/statefulset.yaml | 116 +++++++++++++++++++ charts/seekdb/values.yaml | 86 ++++++++++++++ 10 files changed, 422 insertions(+) create mode 100644 .github/workflows/release-charts.yml create mode 100644 charts/seekdb/Chart.yaml create mode 100644 charts/seekdb/templates/NOTES.txt create mode 100644 charts/seekdb/templates/_helpers.tpl create mode 100644 charts/seekdb/templates/secret.yaml create mode 100644 charts/seekdb/templates/service-obshell.yaml create mode 100644 charts/seekdb/templates/service-sql.yaml create mode 100644 charts/seekdb/templates/serviceaccount.yaml create mode 100644 charts/seekdb/templates/statefulset.yaml create mode 100644 charts/seekdb/values.yaml diff --git a/.github/workflows/release-charts.yml b/.github/workflows/release-charts.yml new file mode 100644 index 0000000..676c4f4 --- /dev/null +++ b/.github/workflows/release-charts.yml @@ -0,0 +1,38 @@ +name: Release Charts + +on: + push: + branches: + - main + - master + - "*-chart" + paths: + - "charts/**" + +jobs: + release: + # specific permissions required for chart-releaser + permissions: + contents: write # to push chart release and create a release + pages: write # to push to gh-pages branch + + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@v3 + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.6.0 + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + CR_SKIP_EXISTING: true diff --git a/charts/seekdb/Chart.yaml b/charts/seekdb/Chart.yaml new file mode 100644 index 0000000..5edb396 --- /dev/null +++ b/charts/seekdb/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: seekdb +description: A Helm chart for seekdb +type: application +version: 0.1.1 +appVersion: "latest" +keywords: + - oceanbase + - seekdb + - database diff --git a/charts/seekdb/templates/NOTES.txt b/charts/seekdb/templates/NOTES.txt new file mode 100644 index 0000000..d81b663 --- /dev/null +++ b/charts/seekdb/templates/NOTES.txt @@ -0,0 +1,30 @@ +1. Get the application URL by running these commands: + +{{- if .Values.service.obshell.enabled }} + Obshell Dashboard: +{{- if contains "NodePort" .Values.service.obshell.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "seekdb.fullname" . }}-obshell) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.obshell.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "seekdb.fullname" . }}-obshell' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "seekdb.fullname" . }}-obshell --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.obshell.port }} +{{- else if contains "ClusterIP" .Values.service.obshell.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "seekdb.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:2886 to access the dashboard" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 2886:2886 +{{- end }} +{{- end }} + +2. Connect to the database: + + export ROOT_PASSWORD=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ include "seekdb.fullname" . }} -o jsonpath="{.data.password}" | base64 -d) + + To connect from inside the cluster: + mysql -h {{ include "seekdb.fullname" . }}-sql.{{ .Release.Namespace }} -P {{ .Values.service.sql.port }} -u root -p$ROOT_PASSWORD + + To connect from outside the cluster (using port-forward): + kubectl port-forward svc/{{ include "seekdb.fullname" . }}-sql 2881:2881 -n {{ .Release.Namespace }} + mysql -h 127.0.0.1 -P 2881 -u root -p$ROOT_PASSWORD \ No newline at end of file diff --git a/charts/seekdb/templates/_helpers.tpl b/charts/seekdb/templates/_helpers.tpl new file mode 100644 index 0000000..a33572c --- /dev/null +++ b/charts/seekdb/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "seekdb.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "seekdb.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "seekdb.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "seekdb.labels" -}} +helm.sh/chart: {{ include "seekdb.chart" . }} +{{ include "seekdb.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "seekdb.selectorLabels" -}} +app.kubernetes.io/name: {{ include "seekdb.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "seekdb.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "seekdb.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/seekdb/templates/secret.yaml b/charts/seekdb/templates/secret.yaml new file mode 100644 index 0000000..eb4f9e2 --- /dev/null +++ b/charts/seekdb/templates/secret.yaml @@ -0,0 +1,20 @@ +{{- $rootPassword := "" -}} +{{- if .Values.seekdb.rootPassword -}} + {{- $rootPassword = .Values.seekdb.rootPassword | b64enc -}} +{{- else -}} + {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace (include "seekdb.fullname" .) ) -}} + {{- if $secretObj -}} + {{- $rootPassword = index $secretObj.data "password" -}} + {{- else -}} + {{- $rootPassword = randAlphaNum 16 | b64enc -}} + {{- end -}} +{{- end -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "seekdb.fullname" . }} + labels: + {{- include "seekdb.labels" . | nindent 4 }} +type: Opaque +data: + password: {{ $rootPassword | quote }} \ No newline at end of file diff --git a/charts/seekdb/templates/service-obshell.yaml b/charts/seekdb/templates/service-obshell.yaml new file mode 100644 index 0000000..fe5da6a --- /dev/null +++ b/charts/seekdb/templates/service-obshell.yaml @@ -0,0 +1,24 @@ +{{- if .Values.service.obshell.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "seekdb.fullname" . }}-obshell + labels: + {{- include "seekdb.labels" . | nindent 4 }} + {{- with .Values.service.obshell.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.obshell.type }} + ports: + - port: {{ .Values.service.obshell.port }} + targetPort: 2886 + protocol: TCP + name: obshell + {{- if and (or (eq .Values.service.obshell.type "NodePort") (eq .Values.service.obshell.type "LoadBalancer")) .Values.service.obshell.nodePort }} + nodePort: {{ .Values.service.obshell.nodePort }} + {{- end }} + selector: + {{- include "seekdb.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/seekdb/templates/service-sql.yaml b/charts/seekdb/templates/service-sql.yaml new file mode 100644 index 0000000..81ded9f --- /dev/null +++ b/charts/seekdb/templates/service-sql.yaml @@ -0,0 +1,24 @@ +{{- if .Values.service.sql.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "seekdb.fullname" . }}-sql + labels: + {{- include "seekdb.labels" . | nindent 4 }} + {{- with .Values.service.sql.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.sql.type }} + ports: + - port: {{ .Values.service.sql.port }} + targetPort: 2881 + protocol: TCP + name: sql + {{- if and (or (eq .Values.service.sql.type "NodePort") (eq .Values.service.sql.type "LoadBalancer")) .Values.service.sql.nodePort }} + nodePort: {{ .Values.service.sql.nodePort }} + {{- end }} + selector: + {{- include "seekdb.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/seekdb/templates/serviceaccount.yaml b/charts/seekdb/templates/serviceaccount.yaml new file mode 100644 index 0000000..345cf90 --- /dev/null +++ b/charts/seekdb/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "seekdb.serviceAccountName" . }} + labels: + {{- include "seekdb.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/seekdb/templates/statefulset.yaml b/charts/seekdb/templates/statefulset.yaml new file mode 100644 index 0000000..d667621 --- /dev/null +++ b/charts/seekdb/templates/statefulset.yaml @@ -0,0 +1,116 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "seekdb.fullname" . }} + labels: + {{- include "seekdb.labels" . | nindent 4 }} +spec: + serviceName: {{ include "seekdb.fullname" . }}-sql + replicas: 1 + selector: + matchLabels: + {{- include "seekdb.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "seekdb.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "seekdb.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: sql + containerPort: 2881 + protocol: TCP + - name: obshell + containerPort: 2886 + protocol: TCP + env: + - name: ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "seekdb.fullname" . }} + key: password + - name: SEEKDB_DATABASE + value: {{ .Values.seekdb.database | quote }} + {{- if .Values.seekdb.config.cpuCount }} + - name: CPU_COUNT + value: {{ .Values.seekdb.config.cpuCount | quote }} + {{- end }} + {{- if .Values.seekdb.config.memoryLimit }} + - name: MEMORY_LIMIT + value: {{ .Values.seekdb.config.memoryLimit | quote }} + {{- end }} + {{- if .Values.seekdb.config.logDiskSize }} + - name: LOG_DISK_SIZE + value: {{ .Values.seekdb.config.logDiskSize | quote }} + {{- end }} + {{- if .Values.seekdb.config.datafileSize }} + - name: DATAFILE_SIZE + value: {{ .Values.seekdb.config.datafileSize | quote }} + {{- end }} + {{- if .Values.seekdb.config.datafileNext }} + - name: DATAFILE_NEXT + value: {{ .Values.seekdb.config.datafileNext | quote }} + {{- end }} + {{- if .Values.seekdb.config.datafileMaxSize }} + - name: DATAFILE_MAXSIZE + value: {{ .Values.seekdb.config.datafileMaxSize | quote }} + {{- end }} + volumeMounts: + - name: data + mountPath: /var/lib/oceanbase + livenessProbe: + tcpSocket: + port: sql + initialDelaySeconds: 2 + periodSeconds: 3 + readinessProbe: + tcpSocket: + port: sql + initialDelaySeconds: 2 + periodSeconds: 3 + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - {{ .Values.persistence.accessMode | quote }} + {{- if .Values.persistence.storageClass }} + {{- if (eq "-" .Values.persistence.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + {{- end }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} diff --git a/charts/seekdb/values.yaml b/charts/seekdb/values.yaml new file mode 100644 index 0000000..89fa065 --- /dev/null +++ b/charts/seekdb/values.yaml @@ -0,0 +1,86 @@ +# Default values for seekdb. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: oceanbase/seekdb + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + sql: + enabled: true + type: ClusterIP + port: 2881 + annotations: {} + obshell: + enabled: true + type: NodePort + port: 2886 + annotations: {} + +persistence: + enabled: true + # storageClass: "-" + accessMode: ReadWriteOnce + size: 50Gi + +resources: + limits: + cpu: 4 + memory: 8Gi + requests: + cpu: 1 + memory: 2Gi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +# SeekDB specific configuration +seekdb: + # The password for the root user. + # If empty, a random 16-character alphanumeric password will be generated. + rootPassword: "" + + # The name of the database to be created at startup + database: "" + + # Environment variables for seekdb configuration + # These correspond to the config file parameters + config: + cpuCount: "" # e.g., 4 + memoryLimit: "" # e.g., 2G + logDiskSize: "" # e.g., 2G + datafileSize: "" # e.g., 2G + datafileNext: "" # e.g., 2G + datafileMaxSize: "" # e.g., 50G From 114c4f1fc567eea360989e25f27fb6a29a730040 Mon Sep 17 00:00:00 2001 From: Chris Sun Date: Fri, 26 Dec 2025 14:18:37 +0800 Subject: [PATCH 27/38] set default value of FORCE_START_WITH_ARGS (#75) --- seekdb/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/seekdb/Dockerfile b/seekdb/Dockerfile index f682dae..c02244e 100644 --- a/seekdb/Dockerfile +++ b/seekdb/Dockerfile @@ -11,5 +11,6 @@ RUN if [[ ${TARGETPLATFORM} == 'linux/amd64' ]] ; then yum install -y https://mi COPY start.sh /root/ RUN chmod +x /root/start.sh ENV REPORTER=docker-seekdb +ENV FORCE_START_WITH_ARGS="true" ENTRYPOINT ["/root/start.sh"] From a8985283ebd581990bf0895c3e50b73e77122f7e Mon Sep 17 00:00:00 2001 From: Chris Sun Date: Mon, 19 Jan 2026 16:51:53 +0800 Subject: [PATCH 28/38] fix #77, add environment variable for root password in the command (#78) --- oceanbase-ce/README.md | 2 +- oceanbase-ce/README_CN.md | 2 +- seekdb/README.md | 2 +- seekdb/README_CN.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/oceanbase-ce/README.md b/oceanbase-ce/README.md index 638935f..5e5fff6 100644 --- a/oceanbase-ce/README.md +++ b/oceanbase-ce/README.md @@ -33,7 +33,7 @@ docker run -p 2881:2881 --name oceanbase-ce -e MODE=slim -d oceanbase/oceanbase- # Execute init SQL scripts after bootstrap, do not change root user's password in SQL scripts. # If you'd like to change root user's password, use variable OB_TENANT_PASSWORD. -docker run -p 2881:2881 --name oceanbase-ce -v {init_sql_folder_path}:/root/boot/init.d -d oceanbase/oceanbase-ce +docker run -p 2881:2881 --name oceanbase-ce -e OB_TENANT_PASSWORD={set_as_your_pwd} -v {init_sql_folder_path}:/root/boot/init.d -d oceanbase/oceanbase-ce ``` The bootstrap procedure may take up to five minutes. Verify the bootstrap completion by running: diff --git a/oceanbase-ce/README_CN.md b/oceanbase-ce/README_CN.md index 6ae8f54..c0da7dd 100644 --- a/oceanbase-ce/README_CN.md +++ b/oceanbase-ce/README_CN.md @@ -33,7 +33,7 @@ docker run -p 2881:2881 --name oceanbase-ce -e MODE=slim -d oceanbase/oceanbase- # 启动后执行初始化SQL脚本，请勿在SQL脚本中更改root用户密码。 # 如果您想更改root用户密码，请使用OB_TENANT_PASSWORD环境变量。 -docker run -p 2881:2881 --name oceanbase-ce -v {init_sql_folder_path}:/root/boot/init.d -d oceanbase/oceanbase-ce +docker run -p 2881:2881 --name oceanbase-ce -e OB_TENANT_PASSWORD={set_as_your_pwd} -v {init_sql_folder_path}:/root/boot/init.d -d oceanbase/oceanbase-ce ``` 启动过程可能需要长达五分钟。通过运行以下命令验证启动是否完成： diff --git a/seekdb/README.md b/seekdb/README.md index 84dd9d3..717a04c 100644 --- a/seekdb/README.md +++ b/seekdb/README.md @@ -24,7 +24,7 @@ docker run -d -p 2881:2881 -p 2886:2886 oceanbase/seekdb # Execute init SQL scripts after bootstrap, you need to mount the directory containing the init scripts then specify the directory in container via environment variable INIT_SCRIPTS_PATH. # Please do not change root user's password in SQL scripts. If you'd like to change root user's password, use environment variable ROOT_PASSWORD. -docker run -d -p 2881:2881 -p 2886:2886 -v {init_sql_folder_path}:/root/boot/init.d -e INIT_SCRIPTS_PATH=/root/boot/init.d oceanbase/seekdb +docker run -d -p 2881:2881 -p 2886:2886 -e ROOT_PASSWORD={set_as_your_pwd} -v {init_sql_folder_path}:/root/boot/init.d -e INIT_SCRIPTS_PATH=/root/boot/init.d oceanbase/seekdb ``` ## Supported Environment Variables diff --git a/seekdb/README_CN.md b/seekdb/README_CN.md index 8840e8b..5d0b52b 100644 --- a/seekdb/README_CN.md +++ b/seekdb/README_CN.md @@ -24,7 +24,7 @@ docker run -d -p 2881:2881 -p 2886:2886 oceanbase/seekdb # 在引导后执行初始化 SQL 脚本，您需要挂载包含初始化脚本的目录，然后通过环境变量 INIT_SCRIPTS_PATH 指定容器中的挂载目录。 # 请勿在 SQL 脚本中更改 root 用户的密码。如果您想更改 root 用户的密码，请使用环境变量 ROOT_PASSWORD。 -docker run -d -p 2881:2881 -p 2886:2886 -v {init_sql_folder_path}:/root/boot/init.d -e INIT_SCRIPTS_PATH=/root/boot/init.d oceanbase/seekdb +docker run -d -p 2881:2881 -p 2886:2886 -e ROOT_PASSWORD={set_as_your_pwd} -v {init_sql_folder_path}:/root/boot/init.d -e INIT_SCRIPTS_PATH=/root/boot/init.d oceanbase/seekdb ``` ## 支持的环境变量 From 60032e060b8877ce72f27877650f63df358f9a82 Mon Sep 17 00:00:00 2001 From: Chris Sun Date: Wed, 28 Jan 2026 15:06:34 +0800 Subject: [PATCH 29/38] seekdb binary name change (#80) --- seekdb/start.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/seekdb/start.sh b/seekdb/start.sh index be6f8ff..688f21b 100755 --- a/seekdb/start.sh +++ b/seekdb/start.sh @@ -7,7 +7,7 @@ WAIT_FOR_PASSWORD_SET_ATTEMPTS=300 WAIT_FOR_SERVICE_READY_ATTEMPTS=300 WAIT_INTERVAL_SECONDS=1 -CONFIG_FILE="/etc/oceanbase/seekdb.cnf" +CONFIG_FILE="/etc/seekdb/seekdb.cnf" # Replace values in config file with environment variables if they are set @@ -36,12 +36,12 @@ if [ -n "$LOG_DISK_SIZE" ]; then fi # Execute the main process -/usr/libexec/oceanbase/scripts/seekdb_systemd_start 2>/dev/null +/usr/libexec/seekdb/scripts/seekdb_systemd_start 2>/dev/null -OBSERVER_CONFIG_FILE="/var/lib/oceanbase/etc/observer.config.bin" +SEEKDB_CONFIG_FILE="/var/lib/oceanbase/etc/seekdb.config.bin" for i in $(seq 1 $WAIT_FOR_CONFIG_FILE_ATTEMPTS); do - if [ -f "$OBSERVER_CONFIG_FILE" ]; then - echo "File '$OBSERVER_CONFIG_FILE' found on attempt #$i." + if [ -f "$SEEKDB_CONFIG_FILE" ]; then + echo "File '$SEEKDB_CONFIG_FILE' found on attempt #$i." break fi if [ $((i % 10)) -eq 0 ]; then @@ -56,7 +56,7 @@ INITIALIZED_FLAG="/var/lib/oceanbase/.initialized" if [ ! -f "$INITIALIZED_FLAG" ]; then # change password using obshell for i in $(seq 1 $WAIT_FOR_PASSWORD_SET_ATTEMPTS); do - curl -X PUT "http://127.0.0.1:2886/api/v1/observer/user/root/password" -d "{\"password\":\"$ROOT_PASSWORD\"}" --unix-socket "/var/lib/oceanbase/run/obshell.sock" + curl -X PUT "http://127.0.0.1:2886/api/v1/seekdb/user/root/password" -d "{\"password\":\"$ROOT_PASSWORD\"}" --unix-socket "/var/lib/oceanbase/run/obshell.sock" EXIT_STATUS=$? if [ $EXIT_STATUS -eq 0 ]; then echo "Command succeeded on attempt #$i." @@ -127,10 +127,11 @@ if [ $# -gt 0 ]; then exit $? fi -echo "Starting observer health check..." -while pgrep observer > /dev/null; do +echo "Seekdb started" +echo "Start seekdb health check loop" +while pgrep seekdb > /dev/null; do sleep 5 done -echo "Observer process not found. Exiting." +echo "Seekdb process not found. Exiting." exit 1 From 999c534164e310ee87f423cd7a0f7f5c81eda5a3 Mon Sep 17 00:00:00 2001 From: Chris Sun Date: Wed, 28 Jan 2026 15:16:58 +0800 Subject: [PATCH 30/38] change config file name in README (#81) --- seekdb/README.md | 4 ++-- seekdb/README_CN.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/seekdb/README.md b/seekdb/README.md index 717a04c..570d1ac 100644 --- a/seekdb/README.md +++ b/seekdb/README.md @@ -42,7 +42,7 @@ Below is a table of supported environment variables for the image: | INIT_SCRIPTS_PATH | The path in the container containing the init scripts. | | SEEKDB_DATABASE | The name of the database to be created at startup. | -If you'd like to modify other seekdb parameters, you can do mount a configuration file into `/etc/oceanbase/seekdb.cnf` in the container, the default configuration file is as follows. +If you'd like to modify other seekdb parameters, you can do mount a configuration file into `/etc/seekdb/seekdb.cnf` in the container, the default configuration file is as follows. ``` datafile_size=2G @@ -58,7 +58,7 @@ log_disk_size=2G The start command should be like this. ``` # **Note:** If you decide to use a configuration file, please don't specify the resource related environment variables. -docker run -d -p 2881:2881 -p 2886:2886 -v {config_file}:/etc/oceanbase/seekdb.cnf oceanbase/seekdb +docker run -d -p 2881:2881 -p 2886:2886 -v {config_file}:/etc/seekdb/seekdb.cnf oceanbase/seekdb ``` ## Data Persistence diff --git a/seekdb/README_CN.md b/seekdb/README_CN.md index 5d0b52b..04d00d7 100644 --- a/seekdb/README_CN.md +++ b/seekdb/README_CN.md @@ -42,7 +42,7 @@ docker run -d -p 2881:2881 -p 2886:2886 -e ROOT_PASSWORD={set_as_your_pwd} -v {i | INIT_SCRIPTS_PATH | 容器中包含初始化脚本的路径。 | | SEEKDB_DATABASE | 启动时要创建的数据库名称。 | -如果您想修改其他 seekdb 参数，可以将配置文件挂载到容器中的 `/etc/oceanbase/seekdb.cnf`，默认配置文件如下。 +如果您想修改其他 seekdb 参数，可以将配置文件挂载到容器中的 `/etc/seekdb/seekdb.cnf`，默认配置文件如下。 ``` datafile_size=2G @@ -58,7 +58,7 @@ log_disk_size=2G 启动命令应如下所示。 ``` # **注意：** 如果您决定使用配置文件，请不要指定与资源相关的环境变量。 -docker run -d -p 2881:2881 -p 2886:2886 -v {config_file}:/etc/oceanbase/seekdb.cnf oceanbase/seekdb +docker run -d -p 2881:2881 -p 2886:2886 -v {config_file}:/etc/seekdb/seekdb.cnf oceanbase/seekdb ``` ## 数据持久化 From b9b1db394ec9bbf80cf1f30f352c1905e0aeabca Mon Sep 17 00:00:00 2001 From: "wangzelin.wzl" Date: Thu, 26 Mar 2026 15:01:20 +0800 Subject: [PATCH 31/38] update seedkb config bin name --- seekdb/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seekdb/start.sh b/seekdb/start.sh index 688f21b..e9c5260 100755 --- a/seekdb/start.sh +++ b/seekdb/start.sh @@ -38,7 +38,7 @@ fi # Execute the main process /usr/libexec/seekdb/scripts/seekdb_systemd_start 2>/dev/null -SEEKDB_CONFIG_FILE="/var/lib/oceanbase/etc/seekdb.config.bin" +SEEKDB_CONFIG_FILE="/var/lib/oceanbase/etc/seekdb.data_version.bin" for i in $(seq 1 $WAIT_FOR_CONFIG_FILE_ATTEMPTS); do if [ -f "$SEEKDB_CONFIG_FILE" ]; then echo "File '$SEEKDB_CONFIG_FILE' found on attempt #$i." From d82bb3da0a07d1e16ae7f4306e0fc653663b4de1 Mon Sep 17 00:00:00 2001 From: "wangzelin.wzl" Date: Mon, 30 Mar 2026 15:52:00 +0800 Subject: [PATCH 32/38] fix seekdb no datafile size option --- seekdb/start.sh | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/seekdb/start.sh b/seekdb/start.sh index e9c5260..6cca03e 100755 --- a/seekdb/start.sh +++ b/seekdb/start.sh @@ -9,30 +9,39 @@ WAIT_INTERVAL_SECONDS=1 CONFIG_FILE="/etc/seekdb/seekdb.cnf" -# Replace values in config file with environment variables if they are set +# Set or append key=value in seekdb.cnf (line may be absent in newer packages) +seekdb_set_cnf() { + local key="$1" + local value="$2" + if [ -f "$CONFIG_FILE" ] && grep -qE "^${key}=" "$CONFIG_FILE"; then + sed -i "s|^${key}=.*|${key}=${value}|" "$CONFIG_FILE" + else + echo "${key}=${value}" >> "$CONFIG_FILE" + fi +} if [ -n "$DATAFILE_SIZE" ]; then - sed -i "s|^datafile_size=.*|datafile_size=$DATAFILE_SIZE|" $CONFIG_FILE + seekdb_set_cnf datafile_size "$DATAFILE_SIZE" fi if [ -n "$DATAFILE_NEXT" ]; then - sed -i "s|^datafile_next=.*|datafile_next=$DATAFILE_NEXT|" $CONFIG_FILE + seekdb_set_cnf datafile_next "$DATAFILE_NEXT" fi if [ -n "$DATAFILE_MAXSIZE" ]; then - sed -i "s|^datafile_maxsize=.*|datafile_maxsize=$DATAFILE_MAXSIZE|" $CONFIG_FILE + seekdb_set_cnf datafile_maxsize "$DATAFILE_MAXSIZE" fi if [ -n "$CPU_COUNT" ]; then - sed -i "s|^cpu_count=.*|cpu_count=$CPU_COUNT|" $CONFIG_FILE + seekdb_set_cnf cpu_count "$CPU_COUNT" fi if [ -n "$MEMORY_LIMIT" ]; then - sed -i "s|^memory_limit=.*|memory_limit=$MEMORY_LIMIT|" $CONFIG_FILE + seekdb_set_cnf memory_limit "$MEMORY_LIMIT" fi if [ -n "$LOG_DISK_SIZE" ]; then - sed -i "s|^log_disk_size=.*|log_disk_size=$LOG_DISK_SIZE|" $CONFIG_FILE + seekdb_set_cnf log_disk_size "$LOG_DISK_SIZE" fi # Execute the main process From 1e88b55c5d41beb5b957746a23c46e3387e84c7f Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Wed, 1 Apr 2026 15:36:22 +0800 Subject: [PATCH 33/38] Build daily seekdb-python artifacts Made-with: Cursor --- .../daily-seekdb-python-artifacts.yml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/daily-seekdb-python-artifacts.yml diff --git a/.github/workflows/daily-seekdb-python-artifacts.yml b/.github/workflows/daily-seekdb-python-artifacts.yml new file mode 100644 index 0000000..aff3221 --- /dev/null +++ b/.github/workflows/daily-seekdb-python-artifacts.yml @@ -0,0 +1,53 @@ +name: daily-seekdb-python-artifacts + +on: + schedule: + # Run once per day (UTC). Version string will include hour. + - cron: "0 0 * * *" + workflow_dispatch: {} + +jobs: + build-wheels: + runs-on: ubuntu-latest + env: + SEEKDB_GIT_TAG: main + steps: + - name: Extract package version (UTC) + run: | + # Example: 2026.04.01.dev00 + echo "PACKAGE_VERSION=$(date -u +'%Y.%m.%d.dev%H')" >> "$GITHUB_ENV" + + - name: Free Disk Space + uses: insightsengineering/disk-space-reclaimer@v1 + with: + tools-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Build wheels (Python 3.11 only) + uses: pypa/cibuildwheel@v3.3.0 + env: + CIBW_BEFORE_BUILD: "pip install setuptools wheel build" + CIBW_BUILD: "cp311-*" + CIBW_SKIP: "*-musllinux_* *-win_* *-macosx_*" + CIBW_ENVIRONMENT: "SEEKDB_GIT_TAG=${{ env.SEEKDB_GIT_TAG }} PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} BUILD_TYPE=release REBUILD=1 SEEKDB_BUILD_LIBRARY=1" + CIBW_TEST_COMMAND: "python {project}/seekdb-python/seekdb_test.py" + CIBW_TEST_REQUIRES: "" + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + with: + package-dir: seekdb-python + output-dir: wheelhouse + + - name: Upload wheels + uses: actions/upload-artifact@v6 + with: + name: wheels-${{ env.PACKAGE_VERSION }}-${{ runner.arch }} + path: wheelhouse/*.whl + retention-days: 30 + compression-level: 0 From 74dc6d3ce636db1d36ad2e37bf73007a84848d7f Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Wed, 1 Apr 2026 16:52:50 +0800 Subject: [PATCH 34/38] Update daily seekdb build to track master Made-with: Cursor --- .github/workflows/daily-seekdb-python-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/daily-seekdb-python-artifacts.yml b/.github/workflows/daily-seekdb-python-artifacts.yml index aff3221..1a1c03b 100644 --- a/.github/workflows/daily-seekdb-python-artifacts.yml +++ b/.github/workflows/daily-seekdb-python-artifacts.yml @@ -10,7 +10,7 @@ jobs: build-wheels: runs-on: ubuntu-latest env: - SEEKDB_GIT_TAG: main + SEEKDB_GIT_TAG: master steps: - name: Extract package version (UTC) run: | From 6a6343aff2998f5ed4964584b3e6898aab8249f2 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Wed, 20 May 2026 11:52:05 +0800 Subject: [PATCH 35/38] update seekdb-python image name (#3) * valid docker image name * update image name --- .github/workflows/release-seekdb-python.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release-seekdb-python.yml b/.github/workflows/release-seekdb-python.yml index 0fa6d69..35783fe 100644 --- a/.github/workflows/release-seekdb-python.yml +++ b/.github/workflows/release-seekdb-python.yml @@ -63,8 +63,7 @@ jobs: CIBW_ENVIRONMENT: "SEEKDB_GIT_TAG=${{ steps.extract_version.outputs.seekdb_version }} PACKAGE_VERSION=${{ steps.extract_version.outputs.package_version }} BUILD_TYPE=release REBUILD=1 SEEKDB_BUILD_LIBRARY=1" CIBW_TEST_COMMAND: "python {project}/seekdb-python/seekdb_test.py" CIBW_TEST_REQUIRES: "" - CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 - CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 + CIBW_MANYLINUX_IMAGE: manylinux_2_28 with: package-dir: seekdb-python output-dir: wheelhouse From ffc11f8092d16e4debd8373ddcbd2c5386878aa4 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Mon, 1 Jun 2026 09:13:47 +0800 Subject: [PATCH 36/38] patch seekdb before build --- .../workflows/daily-seekdb-python-artifacts.yml | 15 ++++++++++++++- seekdb-python/build_python_embed.diff | 13 +++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 seekdb-python/build_python_embed.diff diff --git a/.github/workflows/daily-seekdb-python-artifacts.yml b/.github/workflows/daily-seekdb-python-artifacts.yml index 1a1c03b..c5a060a 100644 --- a/.github/workflows/daily-seekdb-python-artifacts.yml +++ b/.github/workflows/daily-seekdb-python-artifacts.yml @@ -33,7 +33,20 @@ jobs: - name: Build wheels (Python 3.11 only) uses: pypa/cibuildwheel@v3.3.0 env: - CIBW_BEFORE_BUILD: "pip install setuptools wheel build" + CIBW_BEFORE_BUILD: > + pip install setuptools wheel build && + bash -c 'set -euo pipefail; + PROJ="{project}"; + SRC="$PROJ/seekdb-source"; + TAG="${SEEKDB_GIT_TAG:-master}"; + if [ ! -d "$SRC/.git" ]; then + mkdir -p "$SRC"; + git -C "$SRC" init; + git -C "$SRC" remote add origin https://github.com/oceanbase/seekdb.git; + git -C "$SRC" fetch --progress --depth=1 origin "$TAG"; + git -C "$SRC" checkout FETCH_HEAD; + fi; + patch -p1 -N -d "$SRC" < "$PROJ/build_python_embed.diff"' CIBW_BUILD: "cp311-*" CIBW_SKIP: "*-musllinux_* *-win_* *-macosx_*" CIBW_ENVIRONMENT: "SEEKDB_GIT_TAG=${{ env.SEEKDB_GIT_TAG }} PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} BUILD_TYPE=release REBUILD=1 SEEKDB_BUILD_LIBRARY=1" diff --git a/seekdb-python/build_python_embed.diff b/seekdb-python/build_python_embed.diff new file mode 100644 index 0000000..0130e8b --- /dev/null +++ b/seekdb-python/build_python_embed.diff @@ -0,0 +1,13 @@ +diff --git a/src/observer/embed/CMakeLists.txt b/src/observer/embed/CMakeLists.txt +index cf8e5a96c25..c7d7720efcc 100644 +--- a/src/observer/embed/CMakeLists.txt ++++ b/src/observer/embed/CMakeLists.txt +@@ -35,7 +35,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Android") + target_link_libraries(embedded_client PRIVATE seekdb_embed_c) + endif() + +-if(BUILD_EMBED_MODE AND OFF) ++if(BUILD_EMBED_MODE AND ON) + # Set target Python version, can be overridden by CMake parameter + if(NOT DEFINED PYTHON_VERSION) + set(PYTHON_VERSION "3.8") From 62cac526318d87e3ecc1a5145741212df226fc38 Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Mon, 1 Jun 2026 09:16:03 +0800 Subject: [PATCH 37/38] patch before build seekdb --- .github/workflows/release-seekdb-python.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-seekdb-python.yml b/.github/workflows/release-seekdb-python.yml index 35783fe..b708b11 100644 --- a/.github/workflows/release-seekdb-python.yml +++ b/.github/workflows/release-seekdb-python.yml @@ -57,7 +57,20 @@ jobs: - name: Build wheels on ${{ matrix.os }} uses: pypa/cibuildwheel@v3.3.0 env: - CIBW_BEFORE_BUILD: "pip install setuptools wheel build" + CIBW_BEFORE_BUILD: > + pip install setuptools wheel build && + bash -c 'set -euo pipefail; + PROJ="{project}"; + SRC="$PROJ/seekdb-source"; + TAG="${SEEKDB_GIT_TAG:-master}"; + if [ ! -d "$SRC/.git" ]; then + mkdir -p "$SRC"; + git -C "$SRC" init; + git -C "$SRC" remote add origin https://github.com/oceanbase/seekdb.git; + git -C "$SRC" fetch --progress --depth=1 origin "$TAG"; + git -C "$SRC" checkout FETCH_HEAD; + fi; + patch -p1 -N -d "$SRC" < "$PROJ/build_python_embed.diff"' CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" CIBW_SKIP: "*-musllinux_* *-win_* *-macosx_*" CIBW_ENVIRONMENT: "SEEKDB_GIT_TAG=${{ steps.extract_version.outputs.seekdb_version }} PACKAGE_VERSION=${{ steps.extract_version.outputs.package_version }} BUILD_TYPE=release REBUILD=1 SEEKDB_BUILD_LIBRARY=1" From 513d2349898df2697b0ac4d973acb642b6fd377d Mon Sep 17 00:00:00 2001 From: wangyunlai Date: Mon, 1 Jun 2026 10:41:19 +0800 Subject: [PATCH 38/38] Fix seekdb patch paths in cibuildwheel workflows. {project} is the repo root, so seekdb-source and build_python_embed.diff live under seekdb-python/. Co-authored-by: Cursor --- .github/workflows/daily-seekdb-python-artifacts.yml | 5 +++-- .github/workflows/release-seekdb-python.yml | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/daily-seekdb-python-artifacts.yml b/.github/workflows/daily-seekdb-python-artifacts.yml index c5a060a..3503cb9 100644 --- a/.github/workflows/daily-seekdb-python-artifacts.yml +++ b/.github/workflows/daily-seekdb-python-artifacts.yml @@ -37,7 +37,8 @@ jobs: pip install setuptools wheel build && bash -c 'set -euo pipefail; PROJ="{project}"; - SRC="$PROJ/seekdb-source"; + PKG="$PROJ/seekdb-python"; + SRC="$PKG/seekdb-source"; TAG="${SEEKDB_GIT_TAG:-master}"; if [ ! -d "$SRC/.git" ]; then mkdir -p "$SRC"; @@ -46,7 +47,7 @@ jobs: git -C "$SRC" fetch --progress --depth=1 origin "$TAG"; git -C "$SRC" checkout FETCH_HEAD; fi; - patch -p1 -N -d "$SRC" < "$PROJ/build_python_embed.diff"' + patch -p1 -N -d "$SRC" < "$PKG/build_python_embed.diff"' CIBW_BUILD: "cp311-*" CIBW_SKIP: "*-musllinux_* *-win_* *-macosx_*" CIBW_ENVIRONMENT: "SEEKDB_GIT_TAG=${{ env.SEEKDB_GIT_TAG }} PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} BUILD_TYPE=release REBUILD=1 SEEKDB_BUILD_LIBRARY=1" diff --git a/.github/workflows/release-seekdb-python.yml b/.github/workflows/release-seekdb-python.yml index b708b11..88dd343 100644 --- a/.github/workflows/release-seekdb-python.yml +++ b/.github/workflows/release-seekdb-python.yml @@ -61,7 +61,8 @@ jobs: pip install setuptools wheel build && bash -c 'set -euo pipefail; PROJ="{project}"; - SRC="$PROJ/seekdb-source"; + PKG="$PROJ/seekdb-python"; + SRC="$PKG/seekdb-source"; TAG="${SEEKDB_GIT_TAG:-master}"; if [ ! -d "$SRC/.git" ]; then mkdir -p "$SRC"; @@ -70,7 +71,7 @@ jobs: git -C "$SRC" fetch --progress --depth=1 origin "$TAG"; git -C "$SRC" checkout FETCH_HEAD; fi; - patch -p1 -N -d "$SRC" < "$PROJ/build_python_embed.diff"' + patch -p1 -N -d "$SRC" < "$PKG/build_python_embed.diff"' CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" CIBW_SKIP: "*-musllinux_* *-win_* *-macosx_*" CIBW_ENVIRONMENT: "SEEKDB_GIT_TAG=${{ steps.extract_version.outputs.seekdb_version }} PACKAGE_VERSION=${{ steps.extract_version.outputs.package_version }} BUILD_TYPE=release REBUILD=1 SEEKDB_BUILD_LIBRARY=1"