diff --git a/pyproject.toml b/pyproject.toml index dcb945b..f343833 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "tiktoken>=0.12.0", "groq>=1.1.1", "scipy>=1.11", + "fastmemory>=0.4.3", ] [project.scripts] diff --git a/scripts/authentic_atf_benchmark.py b/scripts/authentic_atf_benchmark.py new file mode 100644 index 0000000..6ba94fc --- /dev/null +++ b/scripts/authentic_atf_benchmark.py @@ -0,0 +1,182 @@ +import os +import time +import json +import re +import string +import pandas as pd +from datasets import load_dataset +import fastmemory +import nltk +from nltk.tokenize import word_tokenize +from nltk.tag import pos_tag +from huggingface_hub import hf_hub_download + +# Ensure required NLTK packages are available +try: + nltk.download('punkt', quiet=True) + nltk.download('punkt_tab', quiet=True) + nltk.download('averaged_perceptron_tagger', quiet=True) + nltk.download('averaged_perceptron_tagger_eng', quiet=True) +except Exception as e: + print(f"Warning: NLTK download issues: {e}") + +STOP_WORDS = {"this", "that", "these", "those", "when", "where", "which", "what", "there", "their", "after", "before", "will", "have", "with", "from", "assistant", "user"} + +def extract_concepts(text): + """Entity/Concept extraction for topological linking.""" + try: + tokens = word_tokenize(text) + tagged = pos_tag(tokens) + nouns = [word.lower() for (word, pos) in tagged if pos.startswith('NN') and word.lower() not in STOP_WORDS] + proper_nouns = [word for (word, pos) in tagged if pos == 'NNP'] + return list(set(nouns[:3] + proper_nouns[:2])) + except: + words = text.translate(str.maketrans('', '', string.punctuation)).split() + return [w.lower() for w in words if len(w) > 5 and w.lower() not in STOP_WORDS][:5] + +def generate_atfs(segments, conversation_id): + """Generates ATFs from conversational segments.""" + atfs = [] + for i, seg in enumerate(segments): + logic_text = seg.strip() + if not logic_text: continue + + concepts = extract_concepts(logic_text) + my_id = f"{conversation_id}_{i}" + + # Action is based on the role/type + role = "Assistant" if "assistant:" in logic_text.lower() else "User" + action = f"Logic_{role}_{concepts[0].title()}" if concepts else f"Dialogue_{role}_{i}" + + # Connections (Edges) + connections = [f"[{conversation_id}]"] + connections.extend([f"[{c}]" for c in concepts]) + + # Sanitize for Rust + sanitized_logic = logic_text.replace('\\', '\\\\').replace('\"', '\\\"').replace('\n', ' ') + + atf = ( + f"## [ID: {my_id}]\n" + f"**Action:** {action}\n" + f"**Input:** {{Data}}\n" + f"**Logic:** {sanitized_logic}\n" + f"**Data_Connections:** {', '.join(connections)}\n" + f"**Access:** Open\n" + f"**Events:** Ingest\n\n" + ) + atfs.append(atf) + return "".join(atfs) + +def run_beam_audit(limit=10): + print("\nšŸš€ Initiating BEAM Forensic Audit (Mohammadta/BEAM 100K)...") + try: + ds = load_dataset("Mohammadta/BEAM", split="100K") + except Exception as e: + print(f"Error loading BEAM: {e}") + return [] + + results = [] + samples = list(ds)[:limit] + + for row in samples: + conv_id = row.get("conversation_id", "unknown") + chat = row.get("chat", []) + + # Flatten turns (Mocking AMB _iter_turns) + turns = [] + for session in chat: + if isinstance(session, list): + for turn in session: + role = turn.get("role", "unknown").capitalize() + content = turn.get("content", "") + turns.append(f"{role}: {content}") + + if not turns: continue + + atf_markdown = generate_atfs(turns, conv_id) + + start_time = time.time() + json_graph = fastmemory.process_markdown(atf_markdown) + latency = (time.time() - start_time) * 1000 + + cluster_count = json_graph.count('"block_type"') + results.append({ + "Dataset": "BEAM-100K", + "Sample_ID": conv_id, + "Nodes": len(turns), + "Clusters": cluster_count, + "Latency_ms": latency + }) + print(f"[BEAM] Processed {conv_id}: {len(turns)} turns -> {cluster_count} clusters in {latency:.2f}ms") + + return results + +def run_personamem_audit(limit=10): + print("\nšŸš€ Initiating PersonaMem Forensic Audit (bowen-upenn/PersonaMem)...") + try: + # PersonaMem contexts are in jsonl files in the hub + local_path = hf_hub_download(repo_id="bowen-upenn/PersonaMem", filename="shared_contexts_32k.jsonl", repo_type="dataset") + contexts = [] + with open(local_path, "r") as f: + for line in f: + entry = json.loads(line) + ctx_id, turns = next(iter(entry.items())) + contexts.append((ctx_id, turns)) + if len(contexts) >= limit: break + except Exception as e: + print(f"Error loading PersonaMem: {e}") + return [] + + results = [] + for ctx_id, turns in contexts: + segments = [] + for t in turns: + role = t.get("role", "unknown") + content = t.get("content", "") + segments.append(f"[{role}] {content}") + + atf_markdown = generate_atfs(segments, ctx_id) + + start_time = time.time() + json_graph = fastmemory.process_markdown(atf_markdown) + latency = (time.time() - start_time) * 1000 + + cluster_count = json_graph.count('"block_type"') + results.append({ + "Dataset": "PersonaMem-32K", + "Sample_ID": ctx_id, + "Nodes": len(turns), + "Clusters": cluster_count, + "Latency_ms": latency + }) + print(f"[PersonaMem] Processed {ctx_id}: {len(turns)} segments -> {cluster_count} clusters in {latency:.2f}ms") + + return results + +def main(): + print("--- FASTMEMORY AUTHENTIC BEAM SOTA AUDIT ---") + all_metrics = [] + + # Run BEAM Audit (The primary correction) + beam_results = run_beam_audit(limit=15) + all_metrics.extend(beam_results) + + # Run PersonaMem Audit + pm_results = run_personamem_audit(limit=10) + all_metrics.extend(pm_results) + + if all_metrics: + df = pd.DataFrame(all_metrics) + df.to_csv("authentic_fastmemory_metrics.csv", index=False) + print("\nāœ… CORRECTED BEAM AUDIT COMPLETE.") + print("-" * 50) + print(f"Total Logic Nodes: {df['Nodes'].sum()}") + print(f"Avg Indexing Latency: {df['Latency_ms'].mean():.2f} ms") + print(f"Total Topological Clusters: {df['Clusters'].sum()}") + print("-" * 50) + print("Final BEAM metrics saved to: authentic_fastmemory_metrics.csv") + else: + print("\nāŒ Audit failed. Check logs.") + +if __name__ == "__main__": + main() diff --git a/scripts/authentic_fastmemory_metrics.csv b/scripts/authentic_fastmemory_metrics.csv new file mode 100644 index 0000000..d2ee2c9 --- /dev/null +++ b/scripts/authentic_fastmemory_metrics.csv @@ -0,0 +1,26 @@ +Dataset,Sample_ID,Nodes,Clusters,Latency_ms +BEAM-100K,1,188,436,622.4467754364014 +BEAM-100K,2,200,397,849.2159843444824 +BEAM-100K,3,194,419,703.1517028808594 +BEAM-100K,4,212,332,1312.8509521484375 +BEAM-100K,5,238,338,1070.5039501190186 +BEAM-100K,6,258,506,1182.3718547821045 +BEAM-100K,7,260,476,1068.4700012207031 +BEAM-100K,8,268,457,1868.4842586517334 +BEAM-100K,9,270,485,1223.215103149414 +BEAM-100K,10,344,567,1160.5098247528076 +BEAM-100K,11,388,549,1108.1349849700928 +BEAM-100K,12,392,677,1368.7989711761475 +BEAM-100K,13,310,505,1391.3640975952148 +BEAM-100K,14,268,497,1322.463035583496 +BEAM-100K,15,272,453,1111.0010147094727 +PersonaMem-32K,e898d03fec683b1cabf29f57287ff66f8a31842543ecef44b56766844c1c1301,183,305,1889.7819519042969 +PersonaMem-32K,1b0b224347aea71887603d63880b90c8d37b1f58073098513b839209034c2f3b,183,289,1499.0079402923584 +PersonaMem-32K,ae5c969c32dafa28ff3f884495f4655de811b061007afaf3307d7b858ff7cfae,171,301,1661.374807357788 +PersonaMem-32K,5c8fb86fe80da5b203e7926407dc3a35f763d32e5891082aaae632210734b5a5,170,295,921.4968681335449 +PersonaMem-32K,aa95cf5880d83a73bb98512a07a64fb873fb24d9dac2bb1862f7c00008632260,160,269,1339.721918106079 +PersonaMem-32K,06f12a0c4085193a32bd1658c5f4b8a5e6e7e1f5221d7169f296130c8d69480d,195,310,1159.999132156372 +PersonaMem-32K,8c336cac503ae78c7fe58a6aef0965963041cd579d1a885db4709293b1853829,213,340,905.2162170410156 +PersonaMem-32K,ad5320ec1416e1e17665cee3d166d459ee29357af2a08f63131443bacc85931a,212,338,1662.8828048706055 +PersonaMem-32K,a9f46aff0bd886c1e45562554ffc4d67fcee974f8cdcd41611e465971692a6f5,168,266,1663.3059978485107 +PersonaMem-32K,cf26537544446b92554000ab50a3c44983a1e0b3de21e9923099792f103d84ef,161,264,1048.8677024841309 diff --git a/scripts/verify_fastmemory.py b/scripts/verify_fastmemory.py new file mode 100644 index 0000000..2e09c39 --- /dev/null +++ b/scripts/verify_fastmemory.py @@ -0,0 +1,74 @@ +import os +import sys +import json +import time + +# ZERO DEPENDENCY MOCK MODELS +class Document: + def __init__(self, id, content, user_id): + self.id = id + self.content = content + self.user_id = user_id + +class Query: + def __init__(self, query): + self.query = query + +try: + import fastmemory +except ImportError: + print("!!! Error: 'fastmemory' package not found.") + print("Please run: pip install fastmemory>=0.4.3") + sys.exit(1) + +def run_isolated_audit(): + print("--- [FORENSIC MODE] FastMemory Engine Audit ---") + + # 0. Engine Health Check + print("[STEP 0] Checking Engine Health...") + test_input = "The quick brown fox jumps over the lazy dog. Cats are independent animals." + try: + res = fastmemory.process_markdown(test_input) + if res == "[]": + print("FAILURE: Engine returned empty graph.") + print("DIAGNOSIS: The embedded rust-louvain binary may not be compatible with your platform.") + print(f" Platform: {sys.platform}, Python: {sys.version}") + print("ACTION: pip install --force-reinstall fastmemory>=0.4.3") + return + print(f"SUCCESS: Engine is responsive (output: {len(res)} chars)") + except Exception as e: + print(f"CRASH: Engine failed: {e}") + return + + # 1. Forensic Payload + docs = [ + Document("doc_company", "FastBuilder.AI is a leader in Sovereign AI.", "audit_user"), + Document("doc_tech", "Our topological memory graphs achieve high precision on BEAM.", "audit_user"), + Document("doc_login", "The master vault code is 1234-AX-99.", "audit_user") + ] + + segments = [doc.content for doc in docs] + full_text = " ".join(segments) + + print("\n[STEP 1] Running Engine Indexing...") + try: + json_graph = fastmemory.process_markdown(full_text) + if json_graph == "[]": + print("FAILURE: Engine returned empty graph [].") + return + print(f"SUCCESS: Graph generated (len: {len(json_graph)})") + except Exception as e: + print(f"CRASH: Engine failed to process input: {e}") + return + + # 2. Content Recovery Check + print("\n[STEP 2] Verifying Topology Structure...") + try: + graph = json.loads(json_graph) + total_nodes = sum(len(block.get("nodes", [])) for block in graph) + print(f"SUCCESS: {len(graph)} clusters, {total_nodes} total nodes") + except json.JSONDecodeError: + print("FAILURE: Engine returned invalid JSON") + +if __name__ == "__main__": + run_isolated_audit() diff --git a/src/memory_bench/memory/__init__.py b/src/memory_bench/memory/__init__.py index 59286ca..ff68c24 100644 --- a/src/memory_bench/memory/__init__.py +++ b/src/memory_bench/memory/__init__.py @@ -8,6 +8,7 @@ from .mem0_cloud import Mem0CloudMemoryProvider from .hybrid_search import HybridSearchMemoryProvider from .supermemory import SupermemoryMemoryProvider +from .fastmemory import FastMemoryProvider REGISTRY: dict[str, type[MemoryProvider]] = { "bm25": BM25MemoryProvider, @@ -22,6 +23,7 @@ "mem0-cloud": Mem0CloudMemoryProvider, "qdrant": HybridSearchMemoryProvider, "supermemory": SupermemoryMemoryProvider, + "fastmemory": FastMemoryProvider, } diff --git a/src/memory_bench/memory/base.py b/src/memory_bench/memory/base.py index cb02224..f7805d4 100644 --- a/src/memory_bench/memory/base.py +++ b/src/memory_bench/memory/base.py @@ -1,3 +1,5 @@ +from __future__ import annotations +from __future__ import annotations import asyncio from abc import ABC, abstractmethod from pathlib import Path diff --git a/src/memory_bench/memory/fastmemory.py b/src/memory_bench/memory/fastmemory.py new file mode 100644 index 0000000..0fdba25 --- /dev/null +++ b/src/memory_bench/memory/fastmemory.py @@ -0,0 +1,242 @@ +from __future__ import annotations +import asyncio +import json +import logging +import re +import os +import sys +import fastmemory +from pathlib import Path +from typing import List, Tuple, Dict, Any, Set + +from ..models import Document +from .base import MemoryProvider + +logger = logging.getLogger(__name__) + +class FastMemoryProvider(MemoryProvider): + name = "fastmemory" + description = "Topological Memory using NLTK concept extraction and Louvain graph clustering via a compiled Rust core." + kind = "local" + provider = "fastbuilder" + link = "https://fastbuilder.ai" + + # Common words to ignore during concept extraction + STOP_WORDS = { + "this", "that", "these", "those", "when", "where", "which", "what", + "there", "their", "after", "before", "will", "have", "with", "from", + "about", "would", "could", "should", "some", "other" + } + + def __init__(self, debug_mode: bool = False): + super().__init__() + self.graphs: Dict[str, List[Dict[str, Any]]] = {} # user_id -> compiled_graph + self.concepts: Dict[str, Set[str]] = {} # user_id -> global_concepts + self.isolation_unit = "conversation" + self.debug_mode = debug_mode or os.getenv("FM_DEBUG") == "1" + self._engine_verified = False + self._verify_engine_health() + + def _verify_engine_health(self): + """Internal check to ensure the Rust engine and NLTK pipeline are working.""" + test_input = "The quick brown fox jumps over the lazy dog. Cats are independent animals." + try: + res = fastmemory.process_markdown(test_input) + if res != "[]" and "block_type" in res: + self._engine_verified = True + else: + self._print_engine_panic("Engine returned empty graph for test input.") + except Exception as e: + self._print_engine_panic(f"Engine crash: {str(e)}") + + def _print_engine_panic(self, detail: str): + """Displays a diagnostic error for environment failures.""" + msg = f""" +################################################################################ +# # +# FASTMEMORY ENGINE: INITIALIZATION FAILED # +# # +################################################################################ + +DETAIL: {detail} + +LIKELY CAUSES: +1. The embedded rust-louvain binary is not compatible with your platform. + Run: file $(python3 -c "import fastmemory; print(fastmemory.__file__)") +2. NLTK data (punkt, averaged_perceptron_tagger_eng) is not installed. + Run: python3 -c "import nltk; nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger_eng')" + +REMEDY: +- Upgrade: pip install --force-reinstall fastmemory>=0.4.3 +- Verify: python3 scripts/verify_fastmemory.py + +################################################################################ +""" + print(msg, file=sys.stderr) + logger.critical(msg) + + def prepare(self, store_dir: Path, unit_ids: set[str] | None = None, reset: bool = True) -> None: + """Prepare local storage if needed. For now, we keep the graph in memory.""" + if reset: + self.graphs = {} + self.concepts = {} + + def _extract_concepts(self, text: str) -> List[str]: + """ + Lightweight entity/concept extraction. + Identifies capitalized words and frequent nouns to build topological connections. + """ + # Extract Capitalized Words (Proper Nouns) + proper_nouns = re.findall(r'\b[A-Z][a-z]{3,}\b', text) + + # Extract potential concepts (words > 5 chars, not in stop words) + words = re.findall(r'\b[a-z]{6,}\b', text.lower()) + concepts = [w for w in words if w not in self.STOP_WORDS] + + # Combine and unique + all_concepts = list(set(proper_nouns + concepts)) + return list(all_concepts)[:5] # Limit to top 5 for dense connectivity + + def _sanitize_logic(self, content: str) -> str: + """ + Sanitize content for Action-Topology Format (ATF). + Escapes newlines and characters that confuse the Rust parser. + """ + if not content: + return "" + # Escape newlines to prevent block termination + content = content.replace("\r\n", " ").replace("\n", " ") + # Escape quotes if necessary (ATF logic is space-delimited usually) + content = content.replace('"', '\\"').strip() + return content + + def _to_atf(self, doc: Document) -> str: + """ + Convert a standard Document to Ontological ATF format. + Builds 'Logic Rooms' based on extracted concepts. + """ + sanitized_content = self._sanitize_logic(doc.content) + concepts = self._extract_concepts(sanitized_content) + + # Build Data_Connections (Graph Edges) + user_id = doc.user_id if doc.user_id else "default_user" + connections = [f"[{user_id}]"] + connections.extend([f"[{c}]" for c in concepts]) + + # Dynamic Action name based on primary concept + primary_concept = concepts[0] if concepts else "Standard" + action_name = f"Process_{primary_concept}" + + # Action-Topology Format (ATF) wrapper + return ( + f"## [ID: {doc.id}]\n" + f"**Action:** {action_name}\n" + f"**Input:** {{Data}}\n" + f"**Logic:** {sanitized_content}\n" + f"**Data_Connections:** {', '.join(connections)}\n" + f"**Access:** Open\n" + f"**Events:** Search\n\n" + ) + + def ingest(self, documents: List[Document]) -> None: + """Ingest documents by compiling them into a topological logic graph.""" + # Group by user_id for isolation + by_user: Dict[str, List[Document]] = {} + for doc in documents: + uid = doc.user_id if doc.user_id else "default_user" + if uid not in by_user: + by_user[uid] = [] + by_user[uid].append(doc) + + for uid, docs in by_user.items(): + atf_payload = "".join([self._to_atf(d) for d in docs]) + + if self.debug_mode: + print(f"\n--- [FM_DEBUG] ATF Payload for {uid} ---") + print(atf_payload) + print("--- [FM_DEBUG] END Payload ---\n") + + try: + logger.info(f"Compiling FastMemory graph for user: {uid} ({len(docs)} docs)") + json_graph_str = fastmemory.process_markdown(atf_payload) + + if os.environ.get("FM_DEBUG") == "1": + print(f"\n--- [FM_DEBUG] ATF Payload for {uid} ---\n{atf_payload}\n--- [FM_DEBUG] END Payload ---") + print(f"\n--- [FM_DEBUG] Raw Engine Return (len: {len(json_graph_str)}) ---\n{json_graph_str}\n--- [FM_DEBUG] END Engine ---") + if "Louvain" in json_graph_str: + print("--- [FM_DEBUG] Louvain clustering detected in engine output ---") + + if json_graph_str == "[]": + logger.error(f"FastMemory engine returned an empty graph for user {uid}.") + logger.error("DIAGNOSTIC: If you do not see '[Louvain]' logs above, the Rust engine failed to initialize.") + logger.error("Possible causes: (1) Python 3.9/3.10 binary mismatch (2) Missing macOS system libraries (3) Malformed ATF structure.") + continue + + graph_data = json.loads(json_graph_str) + + if uid not in self.graphs: + self.graphs[uid] = [] + + # FastMemory returns a list of clusters (blocks) + self.graphs[uid].extend(graph_data) + except Exception as e: + logger.error(f"FastMemory Ingestion Error for {uid}: {e}") + if self.debug_mode: + print(f"!!! [FM_DEBUG] INGESTION ERROR: {e}") + + def retrieve(self, query: str, k: int = 10, user_id: str | None = None, query_timestamp: str | None = None) -> Tuple[List[Document], Dict | None]: + """Retrieve top-k relevant documents using topological search.""" + uid = user_id if user_id else "default_user" + if uid not in self.graphs or not self.graphs[uid]: + if self.debug_mode: + print(f"--- [FM_DEBUG] Search failed: Graph for user {uid} is empty. ---") + return [], None + + query_terms = set(query.lower().split()) + query_concepts = set(self._extract_concepts(query)) + + scored_nodes = [] + + # Search through all clusters/nodes in the user's graph + for cluster in self.graphs[uid]: + for node in cluster.get("nodes", []): + # Extract logic and metadata + logic = node.get("logic", "").lower() + node_id = node.get("id", "").lower() + action = node.get("action", "").lower() + + # Data Connections (Topological Edges) + # We prioritize nodes that share 'Concepts' with the query + connections = str(node.get("data_connections", "")).lower() + + score = 0 + for term in query_terms: + if term in logic: + score += 1 + if term in node_id: + score += 5 # High weight for ID matches (NIAH success) + if term in action: + score += 2 + + # Topological Boost: If the query and node share a concept link + for concept in query_concepts: + if concept.lower() in connections: + score += 10 # Massive boost for conceptual alignment + + if score > 0: + scored_nodes.append((score, node)) + + # Sort by score desc and take top k + scored_nodes.sort(key=lambda x: x[0], reverse=True) + top_k = scored_nodes[:k] + + results = [] + for score, node in top_k: + results.append(Document( + id=node.get("id", "unknown"), + content=node.get("logic", ""), + user_id=uid, + meta={"fastmemory_score": score, "cluster_type": cluster.get("block_type")} + )) + + return results, {"total_nodes_searched": sum(len(c.get("nodes", [])) for c in self.graphs[uid])} diff --git a/src/memory_bench/models.py b/src/memory_bench/models.py index 6bedd05..87ebe52 100644 --- a/src/memory_bench/models.py +++ b/src/memory_bench/models.py @@ -1,3 +1,4 @@ +from __future__ import annotations from dataclasses import dataclass, field