#+TITLE: TokenViz: Token Usage Visualization
#+AUTHOR: Jason Walsh
#+DATE: [2024-11-05]
#+PROPERTY: header-args:bash :mkdirp t
#+PROPERTY: header-args:makefile :mkdirp t
#+PROPERTY: header-args:mermaid :mkdirp t :exports both
#+PROPERTY: header-args :tangle yes
#+STARTUP: showall

* Overview

TokenViz provides lightweight token usage visualization for LLM systems using Unix tools.
This simulates what would typically be monitored through enterprise observability platforms,
making it useful for local development and pattern analysis.

* Dashboard Preview

#+CAPTION: TokenViz Dashboard showing real-time token usage visualization
#+NAME: fig:tokenviz-dashboard
[[file:tokenviz_20241106_104613.png]]

* Quick Start

#+begin_src bash
# Launch dashboard (local)
make dashboard

# For container environments
DISPLAY=:1 make dashboard

# View status
make status

# Clean up
make stop
#+end_src

* Architecture

** System Overview
#+begin_src mermaid :file docs/images/architecture.png :tangle docs/architecture.mmd
flowchart TD
    subgraph Generators["Token Generators"]
        G1["Generator 1\n(Random 0-3000)"]
        G2["Generator 2\n(Random 0-3000)"]
        G3["Generator 3\n(Random 0-3000)"]
    end

    subgraph Storage["Storage Layer"]
        L1["/tmp/tokenload/gen1.log"]
        L2["/tmp/tokenload/gen2.log"]
        L3["/tmp/tokenload/gen3.log"]
    end

    subgraph Processing["Aggregation Layer"]
        A1["Aggregator\n(Sum all inputs)"]
        P1["Named Pipe\n/tmp/tokenload_pipe"]
        D1["Data File\n/tmp/tokenload_data"]
    end

    subgraph Display["Visualization Layer"]
        V1["tmux pane 1\nGenerator Logs"]
        V2["tmux pane 2\nTotal Usage"]
        V3["tmux pane 3\nxload Graph"]
    end

    G1 --> L1
    G2 --> L2
    G3 --> L3
    L1 & L2 & L3 --> A1
    A1 --> P1
    A1 --> D1
    L1 & L2 & L3 --> V1
    D1 --> V2
    P1 --> V3
#+end_src

** Data Flow
#+begin_src mermaid :file docs/images/dataflow.png :tangle docs/dataflow.mmd
sequenceDiagram
    participant G as Generators
    participant L as Log Files
    participant A as Aggregator
    participant P as Named Pipe
    participant D as Display

    loop Every Second
        G->>L: Write random token counts
        L->>A: Read latest values
        A->>P: Write sum to pipe
        A->>D: Update display
    end
#+end_src

* Implementation
** Configuration
#+begin_src makefile :tangle Makefile
# TokenViz Configuration
SHELL := /bin/bash
.PHONY: all clean test dashboard stop setup generators aggregator test-tmux test-xload status logs kill-all restart

# Check if we're in a container and set DISPLAY accordingly
CONTAINER_CHECK := $(shell test -f /.dockerenv && echo 1 || echo 0)
ifeq ($(CONTAINER_CHECK),1)
	XDISPLAY := :1
else
	XDISPLAY := :0
endif

# Paths
PIPE := /tmp/tokenload_pipe
DATA := /tmp/tokenload_data
LOGDIR := /tmp/tokenload
SESSION := tokenviz

# Default target
.DEFAULT_GOAL := help

help: ## Show this help message
	@echo 'Usage: make [target]'
	@echo
	@echo 'Targets:'
	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
#+end_src

** Core Components
*** Setup
#+begin_src makefile :tangle Makefile
setup: ## Setup
	@echo "Setting up directories and files..."
	@rm -rf $(LOGDIR) || true
	@mkdir -p $(LOGDIR)
	@rm -f $(PIPE) || true
	@mkfifo $(PIPE)
	@touch $(DATA)
	@for i in 1 2 3; do echo "Initializing gen$$i..." > $(LOGDIR)/gen$$i.log; done
#+end_src

*** Generators
#+begin_src makefile :tangle Makefile
generators: setup ## Generators
	@for i in 1 2 3; do \
		( \
			while true; do \
				if [ -d "$(LOGDIR)" ]; then \
					echo "gen$$i: $$((RANDOM % 3000))" >> "$(LOGDIR)/gen$$i.log"; \
				else \
					exit 0; \
				fi; \
				sleep 1; \
			done \
		) & \
	done

aggregator: setup ## Aggregator
	@( \
		while true; do \
			if [ -d "$(LOGDIR)" ]; then \
				TOTAL=0; \
				for f in $(LOGDIR)/gen*.log; do \
					if [ -f "$$f" ]; then \
						VAL=$$(tail -n1 "$$f" 2>/dev/null | grep -o '[0-9]*$$' || echo 0); \
						TOTAL=$$((TOTAL + VAL)); \
					fi; \
				done; \
				echo "$$TOTAL" > "$(PIPE)" 2>/dev/null || exit 0; \
				echo "[`date '+%H:%M:%S'`] Total: $$TOTAL" > "$(DATA)" 2>/dev/null || exit 0; \
			else \
				exit 0; \
			fi; \
			sleep 1; \
		done \
	) &
#+end_src

** Process Management
*** Dashboard
#+begin_src makefile :tangle Makefile
dashboard: setup ## Dashboard
	@echo "Starting dashboard with DISPLAY=$(XDISPLAY)..."
	@tmux new-session -d -s $(SESSION) -n 'TokenViz' \; \
		split-window -h \; \
		split-window -h \; \
		select-layout even-horizontal \; \
		send-keys -t 0 "while true; do clear; tail -n 10 $(LOGDIR)/gen*.log 2>/dev/null || echo 'Waiting for data...'; sleep 1; done" C-m \; \
		send-keys -t 1 "while true; do clear; tail -n 10 $(DATA) 2>/dev/null || echo 'Waiting for data...'; sleep 1; done" C-m \; \
		send-keys -t 2 "DISPLAY=$(XDISPLAY) xload -geometry 400x200+100+100 -bg black -fg green -scale 5 < $(PIPE)" C-m \; \
		select-pane -t 0
	@echo "Starting generators..."
	@$(MAKE) generators
	@echo "Starting aggregator..."
	@$(MAKE) aggregator
	@echo "Attaching to session..."
	@tmux attach -t $(SESSION)
#+end_src

*** Process Control
#+begin_src makefile :tangle Makefile
stop: ## Stop all processes
	@echo "Stopping all processes..."
	@pkill -f "/bin/bash.*while true.*gen" 2>/dev/null || true
	@pkill -f "while true.*TOTAL" 2>/dev/null || true
	@tmux kill-session -t $(SESSION) 2>/dev/null || true
	@rm -f $(PIPE) $(DATA) 2>/dev/null || true
	@rm -rf $(LOGDIR) 2>/dev/null || true
	@echo "All processes stopped"

kill-all: ## Emergency cleanup
	@echo "Emergency cleanup in progress..."
	@ps ax | grep "gen.*RANDOM" | grep -v grep | awk '{print $$1}' | xargs kill -9 2>/dev/null || true
	@pkill -f "while true.*TOTAL" 2>/dev/null || true
	@echo "Emergency cleanup complete"

restart: stop dashboard ## Restart all services
#+end_src

** Utility Functions
#+begin_src makefile :tangle Makefile
status: ## Status
	@echo "TokenViz Status:"
	@echo "---------------"
	@echo "Environment: $$([ $(CONTAINER_CHECK) -eq 1 ] && echo 'Container' || echo 'Local')"
	@echo "Display: $(XDISPLAY)"
	@echo "\nGenerator processes:"
	@ps ax | grep "while true.*gen" | grep -v grep || echo "No generators running"
	@echo "\nAggregator process:"
	@ps ax | grep "while true.*TOTAL" | grep -v grep || echo "No aggregator running"
	@echo "\nTmux session:"
	@tmux has-session -t $(SESSION) 2>/dev/null && echo "Session $(SESSION) is running" || echo "No session running"

logs: ## Logs
	@echo "Last 5 lines from each generator:"
	@for i in 1 2 3; do \
		echo "\nGenerator $$i:"; \
		tail -n 5 "$(LOGDIR)/gen$$i.log" 2>/dev/null || echo "No log file"; \
	done
	@echo "\nLast 5 lines from aggregator:"
	@tail -n 5 "$(DATA)" 2>/dev/null || echo "No aggregator data"

test-display: ## Test display
	@echo "Container detection: $(CONTAINER_CHECK)"
	@echo "Using DISPLAY=$(XDISPLAY)"
	@echo "Testing X11 connection..."
	@if DISPLAY=$(XDISPLAY) xdpyinfo >/dev/null 2>&1; then \
		echo "X11 connection successful"; \
	else \
		echo "X11 connection failed"; \
		exit 1; \
	fi
#+end_src

** Container Support
*** Test Display
#+begin_src makefile :tangle Makefile
test-display: ## Test display
	@echo "Container detection: $(CONTAINER_CHECK)"
	@echo "Using DISPLAY=$(XDISPLAY)"
	@echo "Testing X11 connection..."
	@if DISPLAY=$(XDISPLAY) xdpyinfo >/dev/null 2>&1; then \
		echo "X11 connection successful"; \
	else \
		echo "X11 connection failed"; \
		exit 1; \
	fi
#+end_src

*** Dockerfile
#+begin_src dockerfile :tangle Dockerfile
FROM ubuntu:22.04

# Install required packages
RUN apt-get update && apt-get install -y \
    tmux \
    x11-apps \
    xauth \
    make \
    && rm -rf /var/lib/apt/lists/*

# Set up working directory
WORKDIR /app

# Copy application files
COPY . .

# Set display for X11
ENV DISPLAY=:1

# Default command
CMD ["make", "dashboard"]
#+end_src

* Cloud Equivalents
** AWS Implementation
#+begin_src mermaid :file docs/images/aws-impl.png
flowchart LR
    subgraph LLMs["LLM Services"]
        L1["Service 1"]
        L2["Service 2"]
        L3["Service 3"]
    end

    subgraph Queue["Message Queue"]
        Q1["SNS Topic\nToken Usage"]
        Q2["SQS Queue\nAggregation"]
    end

    subgraph Monitor["Monitoring"]
        M1["CloudWatch\nMetrics"]
        M2["CloudWatch\nDashboard"]
    end

    L1 & L2 & L3 --> Q1
    Q1 --> Q2
    Q2 --> M1
    M1 --> M2
#+end_src

** Kafka Implementation
#+begin_src mermaid :file docs/images/kafka-impl.png
flowchart LR
    subgraph LLMs["LLM Services"]
        L1["Service 1"]
        L2["Service 2"]
        L3["Service 3"]
    end

    subgraph Kafka["Kafka Cluster"]
        K1["Topic: token-usage"]
        K2["Topic: aggregated-usage"]
    end

    subgraph Process["Processing"]
        P1["Kafka Streams\nAggregation"]
    end

    subgraph Monitor["Monitoring"]
        M1["Metrics API"]
        M2["Dashboard"]
    end

    L1 & L2 & L3 --> K1
    K1 --> P1
    P1 --> K2
    K2 --> M1
    M1 --> M2
#+end_src

** Prometheus/Grafana Implementation
#+begin_src mermaid :file docs/images/prom-impl.png
flowchart LR
    subgraph LLMs["LLM Services"]
        L1["Service 1\n/metrics"]
        L2["Service 2\n/metrics"]
        L3["Service 3\n/metrics"]
    end

    subgraph Collect["Collection"]
        C1["Prometheus\nServer"]
    end

    subgraph Visual["Visualization"]
        V1["Grafana\nDashboard"]
    end

    L1 & L2 & L3 --> C1
    C1 --> V1
#+end_src

* Contributing

#+begin_src markdown :tangle CONTRIBUTING.md
# Contributing to TokenViz

## Development Setup

1. Fork and clone the repository
2. Ensure XQuartz is installed (macOS)
3. Run tests: `make test`
4. Submit PR with clear description

## Container Development

```bash
# Build container
docker build -t tokenviz .

# Run with X11 socket mounted
docker run -v /tmp/.X11-unix:/tmp/.X11-unix tokenviz
```

## Testing
- Run `make test-display` to verify X11 setup
- Run `make test` for full test suite
- Ensure clean shutdown with `make stop`
#+end_src

* File Properties
# Local Variables:
# org-confirm-babel-evaluate: nil
# org-src-preserve-indentation: t
# org-edit-src-content-indentation: 0
# whitespace-style: (face tabs spaces trailing lines space-before-tab newline indentation empty space-after-tab space-mark tab-mark newline-mark)
# whitespace-mode: t
# End: