From 4a2524350d67ea58befdfd8c12e969a0514b6a53 Mon Sep 17 00:00:00 2001 From: Jonathan Stack Date: Tue, 22 Sep 2026 19:17:10 -0700 Subject: [PATCH 1/6] docs(experiments): add PyTorch Jacks or Better video poker example Train a small PyTorch network to imitate an exact hold-EV calculator for 9/6 Jacks or Better, logging training and evaluation to W&B. The training dataset is not committed. generate_dataset.py builds it (about two minutes for 1M hands), and the scripts take its path explicitly: python generate_dataset.py data/hands.npz python train.py --dataset data/hands.npz data/dataset.py only loads datasets; generation defaults live in data/generate.py. --- .../pytorch-video-poker-bot/.gitignore | 9 + .../pytorch/pytorch-video-poker-bot/README.md | 70 ++++++ .../pytorch-video-poker-bot/data/dataset.py | 87 +++++++ .../pytorch-video-poker-bot/data/generate.py | 59 +++++ .../pytorch/pytorch-video-poker-bot/ev.py | 129 +++++++++++ .../pytorch-video-poker-bot/evaluate.py | 55 +++++ .../pytorch/pytorch-video-poker-bot/game.py | 181 +++++++++++++++ .../generate_dataset.py | 28 +++ .../jacks_or_better.py | 116 ++++++++++ .../pytorch/pytorch-video-poker-bot/model.py | 218 ++++++++++++++++++ .../pytorch-video-poker-bot/requirements.txt | 3 + .../pytorch/pytorch-video-poker-bot/train.py | 112 +++++++++ 12 files changed, 1067 insertions(+) create mode 100644 examples/pytorch/pytorch-video-poker-bot/.gitignore create mode 100644 examples/pytorch/pytorch-video-poker-bot/README.md create mode 100644 examples/pytorch/pytorch-video-poker-bot/data/dataset.py create mode 100644 examples/pytorch/pytorch-video-poker-bot/data/generate.py create mode 100644 examples/pytorch/pytorch-video-poker-bot/ev.py create mode 100644 examples/pytorch/pytorch-video-poker-bot/evaluate.py create mode 100644 examples/pytorch/pytorch-video-poker-bot/game.py create mode 100644 examples/pytorch/pytorch-video-poker-bot/generate_dataset.py create mode 100644 examples/pytorch/pytorch-video-poker-bot/jacks_or_better.py create mode 100644 examples/pytorch/pytorch-video-poker-bot/model.py create mode 100644 examples/pytorch/pytorch-video-poker-bot/requirements.txt create mode 100644 examples/pytorch/pytorch-video-poker-bot/train.py diff --git a/examples/pytorch/pytorch-video-poker-bot/.gitignore b/examples/pytorch/pytorch-video-poker-bot/.gitignore new file mode 100644 index 00000000..df1e98b2 --- /dev/null +++ b/examples/pytorch/pytorch-video-poker-bot/.gitignore @@ -0,0 +1,9 @@ +.venv/ +__pycache__/ +*.py[cod] +.DS_Store +checkpoints/ +wandb/ +.env +*.pt +data/*.npz diff --git a/examples/pytorch/pytorch-video-poker-bot/README.md b/examples/pytorch/pytorch-video-poker-bot/README.md new file mode 100644 index 00000000..3cdefe74 --- /dev/null +++ b/examples/pytorch/pytorch-video-poker-bot/README.md @@ -0,0 +1,70 @@ +# Jacks or Better video poker bot + +Train a small PyTorch network to play **9/6 Jacks or Better**, and log +training + evaluation to [Weights & Biases](https://wandb.ai). + +## Video poker, briefly + +You are dealt five cards, choose which to hold (32 possible hold patterns), +draw replacements for the rest, and get paid from a fixed paytable. + +**Jacks or Better** pays for a pair of Jacks or better, two pair, and the +usual poker hands above that. This example uses the common **9/6** paytable +(full house pays 9×, flush pays 6×, with a max-coin royal bonus). + +Optimal play returns about **99.5%** of money wagered. The network here is +trained to imitate an exact expected-value calculator, so it can get close +to that strategy chart. + +## Setup + +```bash +cd examples/pytorch/pytorch-video-poker-bot +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +wandb login +``` + +Then generate the training dataset (1M random deals labeled with exact hold +EVs, deterministic with seed 42). It takes a couple of minutes: + +```bash +python generate_dataset.py data/hands.npz +``` + +## What to run + +| Command | Purpose | +| --- | --- | +| `python generate_dataset.py PATH` | Build the EV-labeled dataset at `PATH` (run first) | +| `python train.py --dataset PATH` | Train on that dataset and log to W&B | +| `python evaluate.py` | Greedy rollout from `checkpoints/hold-network.pt` | + +```bash +python generate_dataset.py data/hands.npz +python train.py --dataset data/hands.npz +python evaluate.py --checkpoint checkpoints/hold-network.pt +``` + +`train.py` is the W&B demo surface: one `wandb.init` context, `run.log` each +epoch, then finish on exit. Useful flag: `--epochs`. For a quicker run, build a +smaller dataset with `python generate_dataset.py data/small.npz --hands 20000`. + +## Layout + +``` +generate_dataset.py # build an EV-labeled .npz dataset +train.py # fit HoldNetwork, log metrics + artifact +evaluate.py # score a checkpoint with greedy play +game.py # cards, ranks, paytable class, deal/hold/draw +jacks_or_better.py # JoB classifier + paytable (+ evaluate_hand wrapper) +ev.py # exact hold EV calculator (uses JoB classify) +model.py # encoding, network, train helpers, play/checkpoint +data/ # dataset loaders (+ generated .npz) +``` + +## How training works + +1. An exact EV calculator labels every hold pattern for each starting hand. +2. A small MLP (`85 → 256 → 256 → 32`) regresses those targets. +3. At play time the network picks the highest-scoring hold and draws. diff --git a/examples/pytorch/pytorch-video-poker-bot/data/dataset.py b/examples/pytorch/pytorch-video-poker-bot/data/dataset.py new file mode 100644 index 00000000..8716ba43 --- /dev/null +++ b/examples/pytorch/pytorch-video-poker-bot/data/dataset.py @@ -0,0 +1,87 @@ +"""Load the Jacks or Better training dataset.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +from model import ( + CARD_FEATURES, + CARDS_PER_HAND, + INPUT_SIZE, + NUM_ACTIONS, + RANK_FEATURES, +) + + +@dataclass(frozen=True) +class DatasetMeta: + game_id: str + bet: int + hand_count: int + + +@dataclass(frozen=True) +class HoldDataset: + metadata: DatasetMeta + cards: np.ndarray + targets: np.ndarray + + @property + def states(self) -> np.ndarray: + return encode_cards(self.cards) + + def split_samples( + self, + *, + seed: int, + train_fraction: float = 0.8, + validation_fraction: float = 0.1, + ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Shuffle into train / validation / test (80/10/10).""" + rng = np.random.default_rng(seed) + order = rng.permutation(len(self.cards)) + train_end = int(len(order) * train_fraction) + val_end = train_end + int(len(order) * validation_fraction) + return order[:train_end], order[train_end:val_end], order[val_end:] + + +def encode_cards(cards: np.ndarray) -> np.ndarray: + """Batch-encode stored card codes into model inputs.""" + card_codes = np.asarray(cards) + if card_codes.ndim != 2 or card_codes.shape[1] != CARDS_PER_HAND: + raise ValueError("cards must have shape (N, 5)") + ranks = card_codes // 4 + suits = card_codes % 4 + rows = np.arange(len(card_codes))[:, None] + positions = np.arange(CARDS_PER_HAND)[None, :] + encoded = np.zeros((len(card_codes), CARDS_PER_HAND, CARD_FEATURES), dtype=np.float32) + encoded[rows, positions, ranks] = 1.0 + encoded[rows, positions, RANK_FEATURES + suits] = 1.0 + return encoded.reshape(len(card_codes), INPUT_SIZE) + + +def load_dataset(path: Path) -> HoldDataset: + if not path.exists(): + raise FileNotFoundError(f"{path} not found; create it with generate_dataset.py {path}") + with np.load(path, allow_pickle=False) as payload: + meta = _read_meta(payload) + cards = payload["cards"] + targets = payload["targets"] + + if cards.shape != (meta.hand_count, CARDS_PER_HAND): + raise ValueError(f"invalid cards shape: {cards.shape}") + if targets.shape != (meta.hand_count, NUM_ACTIONS): + raise ValueError(f"invalid targets shape: {targets.shape}") + + return HoldDataset(metadata=meta, cards=cards, targets=targets) + + +def _read_meta(payload: np.lib.npyio.NpzFile) -> DatasetMeta: + return DatasetMeta( + game_id=str(payload["game"][0]), + bet=int(payload["bet"][0]), + hand_count=int(payload["hand_count"][0]), + ) diff --git a/examples/pytorch/pytorch-video-poker-bot/data/generate.py b/examples/pytorch/pytorch-video-poker-bot/data/generate.py new file mode 100644 index 00000000..ad26822e --- /dev/null +++ b/examples/pytorch/pytorch-video-poker-bot/data/generate.py @@ -0,0 +1,59 @@ +"""Build the Jacks or Better training dataset (1M random deals, EV-labeled).""" + +from __future__ import annotations + +import random +import time +from pathlib import Path + +import numpy as np + +from ev import hold_expected_values +from game import BET, card_to_code, make_deck +from jacks_or_better import GAME_ID, PAYTABLE +from model import NUM_ACTIONS, encode_hand + +DEFAULT_HANDS = 1_000_000 +DEFAULT_SEED = 42 + + +def generate_dataset( + path: Path, + *, + hands: int = DEFAULT_HANDS, + seed: int = DEFAULT_SEED, + force: bool = False, +) -> Path: + if path.exists() and not force: + raise FileExistsError(f"refusing to overwrite {path}; pass --force") + + deck = make_deck() + rng = random.Random(seed) + cards = np.empty((hands, 5), dtype=np.uint8) + targets = np.empty((hands, NUM_ACTIONS), dtype=np.float32) + + started = time.perf_counter() + print(f"labeling {hands:,} random deals", flush=True) + for index in range(hands): + encoded = encode_hand(rng.sample(deck, 5)) + cards[index] = [card_to_code(card) for card in encoded.canonical_cards] + values = np.asarray( + hold_expected_values(encoded.canonical_cards, PAYTABLE), + dtype=np.float32, + ) + targets[index] = values / BET + if (index + 1) % 100_000 == 0 or index + 1 == hands: + print(f"labeled {index + 1:,}/{hands:,}", flush=True) + + path.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed( + path, + cards=cards, + targets=targets, + game=np.asarray([GAME_ID]), + bet=np.asarray([BET], dtype=np.uint8), + hand_count=np.asarray([hands], dtype=np.uint32), + seed=np.asarray([seed], dtype=np.uint64), + ) + print(f"wrote {path} ({path.stat().st_size / 1e6:.1f} MB) in {time.perf_counter() - started:.1f}s") + return path diff --git a/examples/pytorch/pytorch-video-poker-bot/ev.py b/examples/pytorch/pytorch-video-poker-bot/ev.py new file mode 100644 index 00000000..97802c79 --- /dev/null +++ b/examples/pytorch/pytorch-video-poker-bot/ev.py @@ -0,0 +1,129 @@ +"""Exact expected-value calculator for hold decisions. + +Uses Jacks or Better's vectorized classifier to precompute subset tables, +then answers EV for each of the 32 hold masks with a handful of lookups. +""" + +from __future__ import annotations + +from itertools import chain, combinations +from math import comb +from typing import Sequence + +import numpy as np + +from game import BET, Card, Paytable, hand_to_codes +from jacks_or_better import NUM_RANK_CLASSES, RANK_CLASSES, classify + +CARDS_PER_HAND = 5 +DECK_SIZE = 52 +TOTAL_HANDS = comb(DECK_SIZE, CARDS_PER_HAND) +ALL_POSITIONS = 0b11111 + +_BINOMIAL = np.array( + [[comb(n, k) for k in range(CARDS_PER_HAND + 1)] for n in range(DECK_SIZE + 1)], + dtype=np.int64, +) + + +def colex_index(sorted_codes: np.ndarray, size: int) -> np.ndarray: + index = np.zeros(sorted_codes.shape[0], dtype=np.int64) + for position in range(size): + index += _BINOMIAL[sorted_codes[:, position], position + 1] + return index + + +def colex_index_scalar(sorted_codes: Sequence[int]) -> int: + return sum(comb(code, position + 1) for position, code in enumerate(sorted_codes)) + + +class EVTables: + def __init__(self, subset_counts: Sequence[np.ndarray], hand_ranks: np.ndarray) -> None: + self._subset_counts = tuple(subset_counts) + self._hand_ranks = hand_ranks + + def counts_for(self, sorted_codes: Sequence[int]) -> np.ndarray: + size = len(sorted_codes) + index = colex_index_scalar(sorted_codes) + if size == CARDS_PER_HAND: + counts = np.zeros(NUM_RANK_CLASSES, dtype=np.int64) + counts[int(self._hand_ranks[index])] = 1 + return counts + return self._subset_counts[size][index].astype(np.int64) + + +def build_tables() -> EVTables: + flat = np.fromiter( + chain.from_iterable(combinations(range(DECK_SIZE), CARDS_PER_HAND)), + dtype=np.int8, + count=TOTAL_HANDS * CARDS_PER_HAND, + ) + hands = flat.reshape(TOTAL_HANDS, CARDS_PER_HAND) + ranks = classify(hands).astype(np.int64) + + hand_ranks = np.zeros(TOTAL_HANDS, dtype=np.uint8) + hand_ranks[colex_index(hands, CARDS_PER_HAND)] = ranks.astype(np.uint8) + + subset_counts: list[np.ndarray] = [] + for size in range(CARDS_PER_HAND): + rows = comb(DECK_SIZE, size) + table = np.zeros(rows * NUM_RANK_CLASSES, dtype=np.int64) + for positions in combinations(range(CARDS_PER_HAND), size): + if size == 0: + index = np.zeros(TOTAL_HANDS, dtype=np.int64) + else: + index = colex_index(hands[:, positions], size) + table += np.bincount( + index * NUM_RANK_CLASSES + ranks, + minlength=rows * NUM_RANK_CLASSES, + ) + subset_counts.append(table.reshape(rows, NUM_RANK_CLASSES).astype(np.uint32)) + return EVTables(subset_counts, hand_ranks) + + +_TABLES: EVTables | None = None + + +def get_tables() -> EVTables: + """Build EV tables once per process; reuse in memory afterward.""" + global _TABLES + if _TABLES is None: + _TABLES = build_tables() + return _TABLES + + +def hold_expected_values(hand: Sequence[Card], paytable: Paytable) -> list[float]: + """Exact expected profit for each of the 32 hold masks (BET credits).""" + tables = get_tables() + codes = hand_to_codes(hand) + payouts = np.array( + [paytable.payout_for_rank(rank) for rank in RANK_CLASSES], + dtype=np.int64, + ) + + subset_payout = [0] * 32 + for subset in range(32): + sorted_codes = sorted( + codes[position] for position in range(5) if subset & (1 << position) + ) + subset_payout[subset] = int(tables.counts_for(sorted_codes) @ payouts) + + expected_values: list[float] = [] + for hold_mask in range(32): + discards = ALL_POSITIONS ^ hold_mask + total = 0 + subset = discards + while True: + sign = -1 if bin(subset).count("1") % 2 else 1 + total += sign * subset_payout[hold_mask | subset] + if subset == 0: + break + subset = (subset - 1) & discards + draws = comb(47, bin(discards).count("1")) + expected_values.append(total / draws - BET) + return expected_values + + +def best_hold_mask(hand: Sequence[Card], paytable: Paytable) -> int: + values = hold_expected_values(hand, paytable) + return int(max(range(32), key=lambda mask: values[mask])) diff --git a/examples/pytorch/pytorch-video-poker-bot/evaluate.py b/examples/pytorch/pytorch-video-poker-bot/evaluate.py new file mode 100644 index 00000000..7957124b --- /dev/null +++ b/examples/pytorch/pytorch-video-poker-bot/evaluate.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Evaluate a trained hold-network checkpoint with greedy play.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import wandb + +from jacks_or_better import make_game +from model import load_checkpoint, play_hands + +CHECKPOINT = Path(__file__).resolve().parent / "checkpoints" / "hold-network.pt" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--checkpoint", type=Path, default=CHECKPOINT) + parser.add_argument("--hands", type=int, default=100_000) + parser.add_argument("--seed", type=int, default=42) + args = parser.parse_args() + + if not args.checkpoint.exists(): + raise SystemExit(f"checkpoint not found: {args.checkpoint}") + + model, config = load_checkpoint(args.checkpoint) + game = make_game() + + print(f"checkpoint={args.checkpoint} hands={args.hands:,}", flush=True) + + with wandb.init( + project="jacks-or-better", + name=f"eval-{args.checkpoint.stem}-{args.hands}", + job_type="evaluation", + config={ + "checkpoint": str(args.checkpoint), + "hands": args.hands, + "seed": args.seed, + **config, + }, + ) as run: + metrics = play_hands( + model=model, + game=game, + hands=args.hands, + seed=args.seed, + ) + run.summary.update(metrics) + run.log(metrics) + print(f"return_pct={metrics['return_pct']:.2f}% profit={int(metrics['profit']):,}") + + +if __name__ == "__main__": + main() diff --git a/examples/pytorch/pytorch-video-poker-bot/game.py b/examples/pytorch/pytorch-video-poker-bot/game.py new file mode 100644 index 00000000..bf55a5ff --- /dev/null +++ b/examples/pytorch/pytorch-video-poker-bot/game.py @@ -0,0 +1,181 @@ +"""Video poker core: cards, ranks, paytable, and the deal/hold/draw loop. + +The bet is always five coins — baked into BET below. +""" + +from __future__ import annotations + +import random +from dataclasses import dataclass, field +from enum import Enum +from typing import Callable, Mapping, Sequence + +BET = 5 # always max-coin Jacks or Better + +RANKS = tuple(range(2, 15)) # 2..14, Ace high +SUITS = (0, 1, 2, 3) +RANK_CHARS = {11: "J", 12: "Q", 13: "K", 14: "A"} +SUIT_CHARS = ("s", "h", "d", "c") + + +@dataclass(frozen=True) +class Card: + rank: int + suit: int + + def __post_init__(self) -> None: + if self.rank not in RANKS: + raise ValueError(f"invalid rank: {self.rank}") + if self.suit not in SUITS: + raise ValueError(f"invalid suit: {self.suit}") + + def __str__(self) -> str: + rank = str(self.rank) if self.rank <= 10 else RANK_CHARS[self.rank] + return f"{rank}{SUIT_CHARS[self.suit]}" + + +def parse_card(text: str) -> Card: + """Parse strings like 'Ah', 'Td', '7s'.""" + text = text.strip() + if len(text) != 2: + raise ValueError(f"invalid card: {text}") + rank_char, suit_char = text[0].upper(), text[1].lower() + rank_map = {"T": 10, **{v: k for k, v in RANK_CHARS.items()}} + if rank_char.isdigit(): + rank = int(rank_char) + else: + rank = rank_map.get(rank_char) + if rank is None or rank not in RANKS: + raise ValueError(f"invalid rank in card: {text}") + try: + suit = SUIT_CHARS.index(suit_char) + except ValueError as exc: + raise ValueError(f"invalid suit in card: {text}") from exc + return Card(rank=rank, suit=suit) + + +def cards_from_strings(values: Sequence[str]) -> list[Card]: + return [parse_card(value) for value in values] + + +def make_deck() -> list[Card]: + return [Card(rank=rank, suit=suit) for suit in SUITS for rank in RANKS] + + +def card_to_code(card: Card) -> int: + """Pack a card into 0..51 (rank-major).""" + return (card.rank - 2) * 4 + card.suit + + +def code_to_card(code: int) -> Card: + return Card(rank=(code // 4) + 2, suit=code % 4) + + +def hand_to_codes(hand: Sequence[Card]) -> list[int]: + return [card_to_code(card) for card in hand] + + +def shuffle_deck(deck: list[Card], rng: random.Random | None = None) -> None: + (rng or random).shuffle(deck) + + +def deal(deck: list[Card], count: int) -> list[Card]: + if count > len(deck): + raise ValueError("not enough cards left in deck") + dealt = deck[:count] + del deck[:count] + return dealt + + +def apply_hold(hand: Sequence[Card], hold_mask: int, deck: list[Card]) -> list[Card]: + if hold_mask < 0 or hold_mask > 31: + raise ValueError("hold_mask must be a 5-bit integer (0-31)") + held = [card for index, card in enumerate(hand) if hold_mask & (1 << index)] + draw_count = 5 - len(held) + return held + (deal(deck, draw_count) if draw_count else []) + + +class HandRank(Enum): + ROYAL_FLUSH = "royal_flush" + STRAIGHT_FLUSH = "straight_flush" + FOUR_OF_A_KIND = "four_of_a_kind" + FULL_HOUSE = "full_house" + FLUSH = "flush" + STRAIGHT = "straight" + THREE_OF_A_KIND = "three_of_a_kind" + TWO_PAIR = "two_pair" + JACKS_OR_BETTER = "jacks_or_better" + NOTHING = "nothing" + + +@dataclass(frozen=True) +class EvaluatedHand: + rank: HandRank + + def __str__(self) -> str: + return self.rank.value + + +@dataclass(frozen=True) +class Paytable: + """Maps a hand rank to total credits paid (for a fixed BET).""" + + name: str + payouts: Mapping[HandRank, int] = field(default_factory=dict) + + def payout_for(self, hand: EvaluatedHand) -> int: + return self.payouts.get(hand.rank, 0) + + def payout_for_rank(self, rank: HandRank) -> int: + return self.payouts.get(rank, 0) + + +@dataclass(frozen=True) +class PlayResult: + initial_hand: tuple[Card, ...] + final_hand: tuple[Card, ...] + evaluated: EvaluatedHand + payout: int + profit: int + + +@dataclass +class VideoPokerGame: + paytable: Paytable + evaluate: Callable[[Sequence[Card]], EvaluatedHand] + + def evaluate_hand(self, cards: Sequence[Card]) -> EvaluatedHand: + return self.evaluate(cards) + + def payout_for_hand(self, cards: Sequence[Card]) -> int: + return self.paytable.payout_for(self.evaluate_hand(cards)) + + def play_hand( + self, + hold_mask: int, + rng: random.Random | None = None, + ) -> PlayResult: + rng = rng or random + deck = make_deck() + shuffle_deck(deck, rng=rng) + return self.play_hand_with_state(deck, deal(deck, 5), hold_mask) + + def play_hand_with_state( + self, + deck: list[Card], + hand: Sequence[Card], + hold_mask: int, + ) -> PlayResult: + if len(hand) != 5: + raise ValueError("expected a 5-card hand") + initial = tuple(hand) + final = tuple(apply_hold(initial, hold_mask, deck)) + evaluated = self.evaluate_hand(final) + payout = self.paytable.payout_for(evaluated) + return PlayResult( + initial_hand=initial, + final_hand=final, + evaluated=evaluated, + payout=payout, + profit=payout - BET, + ) diff --git a/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py b/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py new file mode 100644 index 00000000..ab1aeba1 --- /dev/null +++ b/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Generate the Jacks or Better training dataset labeled with exact hold EVs.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from data.generate import DEFAULT_HANDS, DEFAULT_SEED, generate_dataset + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path, help="where to write the .npz dataset") + parser.add_argument("--hands", type=int, default=DEFAULT_HANDS) + parser.add_argument("--seed", type=int, default=DEFAULT_SEED) + parser.add_argument("--force", action="store_true", help="overwrite existing file") + args = parser.parse_args() + generate_dataset( + args.output, + hands=args.hands, + seed=args.seed, + force=args.force, + ) + + +if __name__ == "__main__": + main() diff --git a/examples/pytorch/pytorch-video-poker-bot/jacks_or_better.py b/examples/pytorch/pytorch-video-poker-bot/jacks_or_better.py new file mode 100644 index 00000000..5bedea49 --- /dev/null +++ b/examples/pytorch/pytorch-video-poker-bot/jacks_or_better.py @@ -0,0 +1,116 @@ +"""9/6 Jacks or Better: ranking, paytable, and game factory. + +`classify` is the single source of truth for hand ranks (vectorized). +`evaluate_hand` is a thin wrapper that returns the richer EvaluatedHand type. +""" + +from __future__ import annotations + +from typing import Sequence + +import numpy as np + +from game import ( + Card, + EvaluatedHand, + HandRank, + Paytable, + VideoPokerGame, + hand_to_codes, +) + +GAME_ID = "jacks_or_better_9_6" + +# Absolute credits paid at BET=5 (royal includes the max-coin bonus). +PAYTABLE = Paytable( + name="9/6 Jacks or Better", + payouts={ + HandRank.ROYAL_FLUSH: 4000, + HandRank.STRAIGHT_FLUSH: 250, + HandRank.FOUR_OF_A_KIND: 125, + HandRank.FULL_HOUSE: 45, + HandRank.FLUSH: 30, + HandRank.STRAIGHT: 20, + HandRank.THREE_OF_A_KIND: 15, + HandRank.TWO_PAIR: 10, + HandRank.JACKS_OR_BETTER: 5, + HandRank.NOTHING: 0, + }, +) + +# Index into this tuple is what `classify` returns. +RANK_CLASSES: tuple[HandRank, ...] = ( + HandRank.NOTHING, + HandRank.JACKS_OR_BETTER, + HandRank.TWO_PAIR, + HandRank.THREE_OF_A_KIND, + HandRank.STRAIGHT, + HandRank.FLUSH, + HandRank.FULL_HOUSE, + HandRank.FOUR_OF_A_KIND, + HandRank.STRAIGHT_FLUSH, + HandRank.ROYAL_FLUSH, +) +NUM_RANK_CLASSES = len(RANK_CLASSES) + +_TEN_RANK = 8 +_JACK_RANK = 9 +_ACE_RANK = 12 + + +def classify(hands: np.ndarray) -> np.ndarray: + """Rank many hands at once. Each row is five card codes; result is RANK_CLASSES index.""" + codes = np.asarray(hands) + if codes.ndim != 2 or codes.shape[1] != 5: + raise ValueError("expected an (N, 5) array of card codes") + codes = np.sort(codes, axis=1) + + ranks = codes // 4 + suits = codes % 4 + r0, r1, r2, r3, r4 = (ranks[:, i] for i in range(5)) + + is_flush = (suits == suits[:, :1]).all(axis=1) + distinct = (r0 < r1) & (r1 < r2) & (r2 < r3) & (r3 < r4) + is_run = distinct & ((r4 - r0) == 4) + is_wheel = distinct & (r3 == 3) & (r4 == _ACE_RANK) + is_straight = is_run | is_wheel + is_royal = is_run & (r0 == _TEN_RANK) + + pair_01, pair_12, pair_23, pair_34 = r0 == r1, r1 == r2, r2 == r3, r3 == r4 + adjacent = ( + pair_01.astype(np.int8) + + pair_12.astype(np.int8) + + pair_23.astype(np.int8) + + pair_34.astype(np.int8) + ) + has_quads = (r0 == r3) | (r1 == r4) + has_trips = (r0 == r2) | (r1 == r3) | (r2 == r4) + is_full_house = (adjacent == 3) & ~has_quads + is_trips = (adjacent == 2) & has_trips + is_two_pair = (adjacent == 2) & ~has_trips + pair_rank = np.where(pair_01, r0, np.where(pair_12, r1, np.where(pair_23, r2, r3))) + is_high_pair = (adjacent == 1) & (pair_rank >= _JACK_RANK) + + out = np.zeros(codes.shape[0], dtype=np.uint8) + out[is_high_pair] = RANK_CLASSES.index(HandRank.JACKS_OR_BETTER) + out[is_two_pair] = RANK_CLASSES.index(HandRank.TWO_PAIR) + out[is_trips] = RANK_CLASSES.index(HandRank.THREE_OF_A_KIND) + out[is_straight] = RANK_CLASSES.index(HandRank.STRAIGHT) + out[is_flush] = RANK_CLASSES.index(HandRank.FLUSH) + out[is_full_house] = RANK_CLASSES.index(HandRank.FULL_HOUSE) + out[has_quads] = RANK_CLASSES.index(HandRank.FOUR_OF_A_KIND) + out[is_flush & is_straight] = RANK_CLASSES.index(HandRank.STRAIGHT_FLUSH) + out[is_flush & is_royal] = RANK_CLASSES.index(HandRank.ROYAL_FLUSH) + return out + + +def evaluate_hand(cards: Sequence[Card]) -> EvaluatedHand: + """Classify one hand; translate the fast ordinal into EvaluatedHand.""" + if len(cards) != 5: + raise ValueError("expected exactly 5 cards") + ordinal = int(classify(np.asarray([hand_to_codes(cards)], dtype=np.uint8))[0]) + return EvaluatedHand(RANK_CLASSES[ordinal]) + + +def make_game() -> VideoPokerGame: + return VideoPokerGame(paytable=PAYTABLE, evaluate=evaluate_hand) diff --git a/examples/pytorch/pytorch-video-poker-bot/model.py b/examples/pytorch/pytorch-video-poker-bot/model.py new file mode 100644 index 00000000..248e9b7c --- /dev/null +++ b/examples/pytorch/pytorch-video-poker-bot/model.py @@ -0,0 +1,218 @@ +"""Hold network: encode a hand, score the 32 holds, train, save, and play. + +A checkpoint is the learned weights (plus a small config so we can rebuild +the network). Everything runs on CPU — the model is tiny. +""" + +from __future__ import annotations + +import random +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Sequence + +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F + +from game import BET, Card, VideoPokerGame, deal, make_deck, shuffle_deck + +CARDS_PER_HAND = 5 +RANK_FEATURES = 13 +SUIT_FEATURES = 4 +CARD_FEATURES = RANK_FEATURES + SUIT_FEATURES +INPUT_SIZE = CARDS_PER_HAND * CARD_FEATURES +NUM_ACTIONS = 1 << CARDS_PER_HAND # 32 hold patterns + + +@dataclass(frozen=True) +class EncodedHand: + values: np.ndarray + canonical_cards: tuple[Card, ...] + + def hold_mask(self, action: int, dealt_hand: Sequence[Card]) -> int: + """Map a canonical action index back onto the physical dealt order.""" + if action < 0 or action >= NUM_ACTIONS: + raise ValueError(f"action must be 0-{NUM_ACTIONS - 1}") + held = { + card + for index, card in enumerate(self.canonical_cards) + if action & (1 << index) + } + mask = 0 + for index, card in enumerate(dealt_hand): + if card in held: + mask |= 1 << index + if len(held) != bin(mask).count("1"): + raise ValueError("canonical cards must all be present in dealt hand") + return mask + + +def encode_hand(hand: Sequence[Card]) -> EncodedHand: + """Encode a hand invariant to deal order and physical suit names.""" + if len(hand) != CARDS_PER_HAND: + raise ValueError("expected exactly 5 cards") + if len(set(hand)) != CARDS_PER_HAND: + raise ValueError("hand contains duplicate cards") + + suit_rank_masks = [0] * SUIT_FEATURES + for card in hand: + suit_rank_masks[card.suit] |= 1 << (card.rank - 2) + suit_order = sorted(range(SUIT_FEATURES), key=lambda s: (-suit_rank_masks[s], s)) + suit_map = [0] * SUIT_FEATURES + for canonical_suit, physical_suit in enumerate(suit_order): + suit_map[physical_suit] = canonical_suit + + best_cards = tuple(sorted(hand, key=lambda c: (-c.rank, suit_map[c.suit]))) + values = np.zeros((CARDS_PER_HAND, CARD_FEATURES), dtype=np.float32) + for index, card in enumerate(best_cards): + values[index, card.rank - 2] = 1.0 + values[index, RANK_FEATURES + suit_map[card.suit]] = 1.0 + return EncodedHand(values=values.reshape(INPUT_SIZE), canonical_cards=best_cards) + + +class HoldNetwork(nn.Module): + """MLP that scores each of the 32 possible hold patterns.""" + + def __init__(self, hidden_size: int = 256) -> None: + super().__init__() + self.hidden_size = hidden_size + self.layers = nn.Sequential( + nn.Linear(INPUT_SIZE, hidden_size), + nn.ReLU(), + nn.Linear(hidden_size, hidden_size), + nn.ReLU(), + nn.Linear(hidden_size, NUM_ACTIONS), + ) + + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + return self.layers(inputs) + + +def save_checkpoint(path: Path, *, model: HoldNetwork, config: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + torch.save({"weights": model.state_dict(), "config": config}, path) + + +def load_checkpoint(path: Path) -> tuple[HoldNetwork, dict[str, Any]]: + payload = torch.load(path, map_location="cpu", weights_only=False) + config = dict(payload.get("config", {})) + # Older checkpoints used "model_state"; prefer "weights". + state = payload.get("weights") or payload["model_state"] + model = HoldNetwork(hidden_size=int(config.get("hidden_size", 256))) + model.load_state_dict(state) + model.eval() + return model, config + + +def choose_hold(model: HoldNetwork, hand: Sequence[Card]) -> int: + """Pick the highest-scoring hold pattern for this hand.""" + encoded = encode_hand(hand) + with torch.no_grad(): + scores = model(torch.from_numpy(encoded.values).unsqueeze(0)) + action = int(scores.argmax(dim=1).item()) + return encoded.hold_mask(action, hand) + + +def play_hands( + *, + model: HoldNetwork, + game: VideoPokerGame, + hands: int, + seed: int = 42, + log_every: int = 10_000, +) -> dict[str, float]: + """Play greedy hands; return return_pct / profit / wagered / payout.""" + rng = random.Random(seed) + wagered = 0 + payout = 0 + for hand_num in range(1, hands + 1): + deck = make_deck() + shuffle_deck(deck, rng=rng) + dealt = deal(deck, 5) + result = game.play_hand_with_state(deck, dealt, choose_hold(model, dealt)) + wagered += BET + payout += result.payout + if hand_num % log_every == 0 or hand_num == hands: + print( + f"hand {hand_num:,}/{hands:,} " + f"return={(payout / wagered) * 100:.2f}% " + f"profit={payout - wagered:,}", + flush=True, + ) + return { + "hands": float(hands), + "bet": float(BET), + "wagered": float(wagered), + "payout": float(payout), + "profit": float(payout - wagered), + "return_pct": (payout / wagered) * 100, + } + + +def train_epoch( + model: HoldNetwork, + optimizer: torch.optim.Optimizer, + *, + states: np.ndarray, + targets: np.ndarray, + train_ids: np.ndarray, + batch_size: int, + rng: np.random.Generator, +) -> float: + model.train() + order = rng.permutation(len(train_ids)) + total_loss = 0.0 + n = 0 + for start in range(0, len(order), batch_size): + ids = train_ids[order[start : start + batch_size]] + loss = F.smooth_l1_loss( + model(torch.from_numpy(states[ids])), + torch.from_numpy(targets[ids]), + ) + optimizer.zero_grad() + loss.backward() + nn.utils.clip_grad_norm_(model.parameters(), max_norm=10.0) + optimizer.step() + total_loss += float(loss.item()) * len(ids) + n += len(ids) + return total_loss / n + + +@torch.no_grad() +def validation_metrics( + model: HoldNetwork, + *, + states: np.ndarray, + targets: np.ndarray, + sample_ids: np.ndarray, + batch_size: int, +) -> dict[str, float]: + """Loss, % optimal action, and expected return vs the EV labels.""" + model.eval() + total_loss = total_regret = total_policy = 0.0 + optimal = n = 0 + for start in range(0, len(sample_ids), batch_size): + ids = sample_ids[start : start + batch_size] + batch = targets[ids] + preds = model(torch.from_numpy(states[ids])) + total_loss += float( + F.smooth_l1_loss( + preds, torch.from_numpy(batch), reduction="sum" + ).item() + ) + chosen = preds.argmax(dim=1).numpy() + best = batch.max(axis=1) + chosen_values = batch[np.arange(len(ids)), chosen] + regrets = best - chosen_values + total_regret += float(regrets.sum()) + total_policy += float(chosen_values.sum()) + optimal += int(np.count_nonzero(regrets <= 1e-7)) + n += len(ids) + return { + "loss": total_loss / (n * targets.shape[1]), + "optimal_action_pct": (optimal / n) * 100, + "mean_ev_regret": total_regret / n, + "expected_return_pct": (1.0 + total_policy / n) * 100, + } diff --git a/examples/pytorch/pytorch-video-poker-bot/requirements.txt b/examples/pytorch/pytorch-video-poker-bot/requirements.txt new file mode 100644 index 00000000..be506701 --- /dev/null +++ b/examples/pytorch/pytorch-video-poker-bot/requirements.txt @@ -0,0 +1,3 @@ +numpy +torch +wandb diff --git a/examples/pytorch/pytorch-video-poker-bot/train.py b/examples/pytorch/pytorch-video-poker-bot/train.py new file mode 100644 index 00000000..c229f509 --- /dev/null +++ b/examples/pytorch/pytorch-video-poker-bot/train.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Train a hold network on exact EV targets and log the run to W&B.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import numpy as np +import torch +import wandb + +from data.dataset import load_dataset +from game import BET +from model import ( + INPUT_SIZE, + NUM_ACTIONS, + HoldNetwork, + save_checkpoint, + train_epoch, + validation_metrics, +) + +CHECKPOINT = Path(__file__).resolve().parent / "checkpoints" / "hold-network.pt" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset", type=Path, required=True, help=".npz from generate_dataset.py") + parser.add_argument("--epochs", type=int, default=20) + parser.add_argument("--batch-size", type=int, default=1024) + parser.add_argument("--lr", type=float, default=1e-3) + parser.add_argument("--hidden-size", type=int, default=256) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--checkpoint", type=Path, default=CHECKPOINT) + args = parser.parse_args() + + torch.manual_seed(args.seed) + np.random.seed(args.seed) + + data = load_dataset(args.dataset) + states = data.states + train_ids, val_ids, _test_ids = data.split_samples(seed=args.seed) + model = HoldNetwork(args.hidden_size) + optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) + rng = np.random.default_rng(args.seed) + + config = { + "epochs": args.epochs, + "batch_size": args.batch_size, + "lr": args.lr, + "hidden_size": args.hidden_size, + "seed": args.seed, + "bet": BET, + "input_size": INPUT_SIZE, + "num_actions": NUM_ACTIONS, + "game": data.metadata.game_id, + "hands": data.metadata.hand_count, + } + + # One place for init → log → finish (finish runs automatically on exit). + with wandb.init( + project="jacks-or-better", + name=f"train-h{args.hidden_size}", + config=config, + ) as run: + best_regret = float("inf") + for epoch in range(1, args.epochs + 1): + train_loss = train_epoch( + model, + optimizer, + states=states, + targets=data.targets, + train_ids=train_ids, + batch_size=args.batch_size, + rng=rng, + ) + val = validation_metrics( + model, + states=states, + targets=data.targets, + sample_ids=val_ids, + batch_size=args.batch_size, + ) + run.log( + { + "epoch": epoch, + "train_loss": train_loss, + "val_loss": val["loss"], + "val_optimal_action_pct": val["optimal_action_pct"], + "val_expected_return_pct": val["expected_return_pct"], + }, + step=epoch, + ) + print( + f"epoch {epoch}/{args.epochs} " + f"loss={train_loss:.4f} " + f"val_optimal={val['optimal_action_pct']:.1f}% " + f"val_return={val['expected_return_pct']:.2f}%" + ) + if val["mean_ev_regret"] < best_regret: + best_regret = val["mean_ev_regret"] + save_checkpoint(args.checkpoint, model=model, config=config) + + artifact = wandb.Artifact(args.checkpoint.stem, type="model") + artifact.add_file(str(args.checkpoint)) + run.log_artifact(artifact) + print(f"saved {args.checkpoint}") + + +if __name__ == "__main__": + main() From d8d4f9a811a35f98fcb8d5b3da4e34596c4a562d Mon Sep 17 00:00:00 2001 From: Jonathan Stack Date: Tue, 22 Sep 2026 19:55:59 -0700 Subject: [PATCH 2/6] docs(experiments): require explicit paths and W&B names in video poker example Every file path, the W&B project, and the run name are now required flags, with no hardcoded defaults: generate_dataset.py --output train.py --dataset --checkpoint --project --run-name evaluate.py --checkpoint --project --run-name The README now leads with the W&B run lifecycle (init, log, artifact, finish), explains video poker in plain terms with the 9/6 Jacks or Better paytable in credits, and describes each file in one line. --- .../pytorch/pytorch-video-poker-bot/README.md | 137 ++++++++++++------ .../pytorch-video-poker-bot/data/dataset.py | 2 +- .../pytorch-video-poker-bot/evaluate.py | 10 +- .../generate_dataset.py | 2 +- .../pytorch/pytorch-video-poker-bot/train.py | 10 +- 5 files changed, 107 insertions(+), 54 deletions(-) diff --git a/examples/pytorch/pytorch-video-poker-bot/README.md b/examples/pytorch/pytorch-video-poker-bot/README.md index 3cdefe74..d73fdbfe 100644 --- a/examples/pytorch/pytorch-video-poker-bot/README.md +++ b/examples/pytorch/pytorch-video-poker-bot/README.md @@ -1,70 +1,123 @@ -# Jacks or Better video poker bot +# Train a video poker bot with PyTorch and W&B -Train a small PyTorch network to play **9/6 Jacks or Better**, and log -training + evaluation to [Weights & Biases](https://wandb.ai). +This example trains a small PyTorch network to play video poker and tracks +it with [Weights & Biases](https://wandb.ai). The poker is just a stand-in +task. The point is to see a complete W&B run: -## Video poker, briefly +1. **`wandb.init()`** starts a run and records its config (hyperparameters + and dataset size). +2. **`run.log()`** sends metrics every epoch, so you can watch training live. +3. **`run.log_artifact()`** uploads the trained checkpoint as a versioned + model artifact. +4. **Finish**: `train.py` opens the run with `with wandb.init(...) as run:`, + so the run finishes automatically when the block exits. Without the + `with` block, you would call `run.finish()` yourself. -You are dealt five cards, choose which to hold (32 possible hold patterns), -draw replacements for the rest, and get paid from a fixed paytable. +`evaluate.py` then starts a second run (`job_type="evaluation"`) that plays +hands with the trained checkpoint and records the final score. -**Jacks or Better** pays for a pair of Jacks or better, two pair, and the -usual poker hands above that. This example uses the common **9/6** paytable -(full house pays 9×, flush pays 6×, with a max-coin royal bonus). +## How video poker works -Optimal play returns about **99.5%** of money wagered. The network here is -trained to imitate an exact expected-value calculator, so it can get close -to that strategy chart. +You are dealt five cards. You choose which of them to hold: none, all five, +or any mix in between (32 possible combinations). The cards you don't hold +are replaced with new ones from the deck, and that is your final hand. Its +reward is looked up in the paytable. + +This example plays **9/6 Jacks or Better**, betting 5 credits per hand: + +| Final hand | Reward (credits) | +| --- | --- | +| Royal flush | 4000 | +| Straight flush | 250 | +| Four of a kind | 125 | +| Full house | 45 | +| Flush | 30 | +| Straight | 20 | +| Three of a kind | 15 | +| Two pair | 10 | +| Pair of jacks or better | 5 | +| Anything else | 0 | + +The name "9/6" comes from the full house paying 9 and the flush paying 6 for +each credit bet. With optimal holds, the rewards average about 99.5% of the +credits wagered. + +The dataset gives each dealt hand the exact expected reward of all 32 hold +choices. The network learns to predict those values, and at play time it +makes the hold with the highest prediction. ## Setup ```bash cd examples/pytorch/pytorch-video-poker-bot -python3 -m venv .venv && source .venv/bin/activate +python3 -m venv .venv +source .venv/bin/activate pip install -r requirements.txt wandb login ``` -Then generate the training dataset (1M random deals labeled with exact hold -EVs, deterministic with seed 42). It takes a couple of minutes: +## Run it + +### 1. Generate the dataset ```bash -python generate_dataset.py data/hands.npz +python generate_dataset.py --output data/hands.npz ``` -## What to run +This deals 1,000,000 random hands and labels each one. It takes about two +minutes and writes a ~44 MB file. For a quick test, add `--hands 20000`. -| Command | Purpose | -| --- | --- | -| `python generate_dataset.py PATH` | Build the EV-labeled dataset at `PATH` (run first) | -| `python train.py --dataset PATH` | Train on that dataset and log to W&B | -| `python evaluate.py` | Greedy rollout from `checkpoints/hold-network.pt` | +### 2. Train ```bash -python generate_dataset.py data/hands.npz -python train.py --dataset data/hands.npz -python evaluate.py --checkpoint checkpoints/hold-network.pt +python train.py \ + --dataset data/hands.npz \ + --checkpoint checkpoints/jacks_or_better_network.pt \ + --project video-poker \ + --run-name train-baseline ``` -`train.py` is the W&B demo surface: one `wandb.init` context, `run.log` each -epoch, then finish on exit. Useful flag: `--epochs`. For a quicker run, build a -smaller dataset with `python generate_dataset.py data/small.npz --hands 20000`. +This takes under a minute on a laptop CPU. After each epoch it saves the +checkpoint if validation improved, and at the end it uploads the checkpoint +as a model artifact. Optional flags: `--epochs` (default 20), `--lr`, +`--batch-size`, `--hidden-size`, `--seed`. -## Layout +### 3. Evaluate +```bash +python evaluate.py \ + --checkpoint checkpoints/jacks_or_better_network.pt \ + --project video-poker \ + --run-name eval-baseline ``` -generate_dataset.py # build an EV-labeled .npz dataset -train.py # fit HoldNetwork, log metrics + artifact -evaluate.py # score a checkpoint with greedy play -game.py # cards, ranks, paytable class, deal/hold/draw -jacks_or_better.py # JoB classifier + paytable (+ evaluate_hand wrapper) -ev.py # exact hold EV calculator (uses JoB classify) -model.py # encoding, network, train helpers, play/checkpoint -data/ # dataset loaders (+ generated .npz) -``` -## How training works +This plays 100,000 hands with the checkpoint (change with `--hands`), which +takes about 10 seconds, and logs the result as a separate run in the same +project. + +## What you will see in W&B + +- **Training run**: config on the Overview tab, and these charts per epoch: + - `train_loss` and `val_loss` + - `val_optimal_action_pct`: how often the network makes the optimal hold + - `val_expected_return_pct`: expected rewards as a percentage of credits + wagered -1. An exact EV calculator labels every hold pattern for each starting hand. -2. A small MLP (`85 → 256 → 256 → 32`) regresses those targets. -3. At play time the network picks the highest-scoring hold and draws. + The checkpoint is under Artifacts. +- **Evaluation run**: in the run summary, `wagered` (credits bet), `payout` + (credits rewarded), `profit` (the difference) and `return_pct` (rewards as + a percentage of credits wagered). + +## Files + +| File | What it does | +| --- | --- | +| `generate_dataset.py` | Builds the labeled training dataset. | +| `train.py` | Trains the network and logs the run to W&B. | +| `evaluate.py` | Plays hands with a trained checkpoint and logs the score to W&B. | +| `model.py` | The network, hand encoding, training and validation steps, and checkpoint save/load. | +| `game.py` | Cards, deck, dealing and drawing. | +| `jacks_or_better.py` | Hand rankings and the paytable. | +| `ev.py` | Calculates the exact expected reward of each hold choice. | +| `data/dataset.py` | Loads a generated dataset for training. | +| `data/generate.py` | Deals and labels the hands for `generate_dataset.py`. | diff --git a/examples/pytorch/pytorch-video-poker-bot/data/dataset.py b/examples/pytorch/pytorch-video-poker-bot/data/dataset.py index 8716ba43..f036b46c 100644 --- a/examples/pytorch/pytorch-video-poker-bot/data/dataset.py +++ b/examples/pytorch/pytorch-video-poker-bot/data/dataset.py @@ -65,7 +65,7 @@ def encode_cards(cards: np.ndarray) -> np.ndarray: def load_dataset(path: Path) -> HoldDataset: if not path.exists(): - raise FileNotFoundError(f"{path} not found; create it with generate_dataset.py {path}") + raise FileNotFoundError(f"{path} not found; create it with generate_dataset.py --output {path}") with np.load(path, allow_pickle=False) as payload: meta = _read_meta(payload) cards = payload["cards"] diff --git a/examples/pytorch/pytorch-video-poker-bot/evaluate.py b/examples/pytorch/pytorch-video-poker-bot/evaluate.py index 7957124b..3ad9d566 100644 --- a/examples/pytorch/pytorch-video-poker-bot/evaluate.py +++ b/examples/pytorch/pytorch-video-poker-bot/evaluate.py @@ -11,12 +11,12 @@ from jacks_or_better import make_game from model import load_checkpoint, play_hands -CHECKPOINT = Path(__file__).resolve().parent / "checkpoints" / "hold-network.pt" - def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--checkpoint", type=Path, default=CHECKPOINT) + parser.add_argument("--checkpoint", type=Path, required=True, help=".pt saved by train.py") + parser.add_argument("--project", required=True, help="W&B project to log to") + parser.add_argument("--run-name", required=True, help="name for this W&B run") parser.add_argument("--hands", type=int, default=100_000) parser.add_argument("--seed", type=int, default=42) args = parser.parse_args() @@ -30,8 +30,8 @@ def main() -> None: print(f"checkpoint={args.checkpoint} hands={args.hands:,}", flush=True) with wandb.init( - project="jacks-or-better", - name=f"eval-{args.checkpoint.stem}-{args.hands}", + project=args.project, + name=args.run_name, job_type="evaluation", config={ "checkpoint": str(args.checkpoint), diff --git a/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py b/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py index ab1aeba1..1418e4e7 100644 --- a/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py +++ b/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py @@ -11,7 +11,7 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("output", type=Path, help="where to write the .npz dataset") + parser.add_argument("--output", type=Path, required=True, help="where to write the dataset (.npz)") parser.add_argument("--hands", type=int, default=DEFAULT_HANDS) parser.add_argument("--seed", type=int, default=DEFAULT_SEED) parser.add_argument("--force", action="store_true", help="overwrite existing file") diff --git a/examples/pytorch/pytorch-video-poker-bot/train.py b/examples/pytorch/pytorch-video-poker-bot/train.py index c229f509..ed383320 100644 --- a/examples/pytorch/pytorch-video-poker-bot/train.py +++ b/examples/pytorch/pytorch-video-poker-bot/train.py @@ -21,18 +21,18 @@ validation_metrics, ) -CHECKPOINT = Path(__file__).resolve().parent / "checkpoints" / "hold-network.pt" - def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--dataset", type=Path, required=True, help=".npz from generate_dataset.py") + parser.add_argument("--checkpoint", type=Path, required=True, help="where to save the trained model (.pt)") + parser.add_argument("--project", required=True, help="W&B project to log to") + parser.add_argument("--run-name", required=True, help="name for this W&B run") parser.add_argument("--epochs", type=int, default=20) parser.add_argument("--batch-size", type=int, default=1024) parser.add_argument("--lr", type=float, default=1e-3) parser.add_argument("--hidden-size", type=int, default=256) parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--checkpoint", type=Path, default=CHECKPOINT) args = parser.parse_args() torch.manual_seed(args.seed) @@ -60,8 +60,8 @@ def main() -> None: # One place for init → log → finish (finish runs automatically on exit). with wandb.init( - project="jacks-or-better", - name=f"train-h{args.hidden_size}", + project=args.project, + name=args.run_name, config=config, ) as run: best_regret = float("inf") From 1cae0c0f7b0ee892db71269f1de43646099b6dec Mon Sep 17 00:00:00 2001 From: Jonathan Stack Date: Tue, 22 Sep 2026 20:23:33 -0700 Subject: [PATCH 3/6] docs(experiments): simplify video poker code and fetch the model from W&B - Fix a train/play mismatch: training encoded real suits while play relabeled them first. There is now one encoder, used by both. - evaluate.py fetches the model with run.use_artifact() (linking it to the training run) and logs running totals every 100 hands. - train.py takes --artifact-name, logs val_mean_regret (used to pick the best checkpoint), and records only real hyperparameters in its config. - Fold data/generate.py into generate_dataset.py and move the loader to dataset.py. Datasets now hold just cards and targets, split 90/10. - Remove unused code across game.py, jacks_or_better.py, ev.py and model.py; share hand-size constants from game.py; rename the model class to Network; checkpoints store only hidden size and weights. --- .../pytorch-video-poker-bot/.gitignore | 1 + .../pytorch/pytorch-video-poker-bot/README.md | 38 +++-- .../pytorch-video-poker-bot/data/dataset.py | 87 ---------- .../pytorch-video-poker-bot/data/generate.py | 59 ------- .../pytorch-video-poker-bot/dataset.py | 48 ++++++ .../pytorch/pytorch-video-poker-bot/ev.py | 40 +++-- .../pytorch-video-poker-bot/evaluate.py | 70 +++++--- .../pytorch/pytorch-video-poker-bot/game.py | 98 ++--------- .../generate_dataset.py | 45 +++-- .../jacks_or_better.py | 8 +- .../pytorch/pytorch-video-poker-bot/model.py | 160 +++++------------- .../pytorch/pytorch-video-poker-bot/train.py | 44 ++--- 12 files changed, 244 insertions(+), 454 deletions(-) delete mode 100644 examples/pytorch/pytorch-video-poker-bot/data/dataset.py delete mode 100644 examples/pytorch/pytorch-video-poker-bot/data/generate.py create mode 100644 examples/pytorch/pytorch-video-poker-bot/dataset.py diff --git a/examples/pytorch/pytorch-video-poker-bot/.gitignore b/examples/pytorch/pytorch-video-poker-bot/.gitignore index df1e98b2..8073a10a 100644 --- a/examples/pytorch/pytorch-video-poker-bot/.gitignore +++ b/examples/pytorch/pytorch-video-poker-bot/.gitignore @@ -7,3 +7,4 @@ wandb/ .env *.pt data/*.npz +artifacts/ diff --git a/examples/pytorch/pytorch-video-poker-bot/README.md b/examples/pytorch/pytorch-video-poker-bot/README.md index d73fdbfe..c1c05ea6 100644 --- a/examples/pytorch/pytorch-video-poker-bot/README.md +++ b/examples/pytorch/pytorch-video-poker-bot/README.md @@ -13,8 +13,9 @@ task. The point is to see a complete W&B run: so the run finishes automatically when the block exits. Without the `with` block, you would call `run.finish()` yourself. -`evaluate.py` then starts a second run (`job_type="evaluation"`) that plays -hands with the trained checkpoint and records the final score. +`evaluate.py` then starts a second run (`job_type="evaluation"`). It fetches +the model artifact from W&B with **`run.use_artifact()`**, which links the +two runs, and logs its score every 100 hands as it plays. ## How video poker works @@ -73,40 +74,46 @@ minutes and writes a ~44 MB file. For a quick test, add `--hands 20000`. python train.py \ --dataset data/hands.npz \ --checkpoint checkpoints/jacks_or_better_network.pt \ + --artifact-name jacks-or-better-network \ --project video-poker \ --run-name train-baseline ``` This takes under a minute on a laptop CPU. After each epoch it saves the checkpoint if validation improved, and at the end it uploads the checkpoint -as a model artifact. Optional flags: `--epochs` (default 20), `--lr`, -`--batch-size`, `--hidden-size`, `--seed`. +to W&B as the model artifact `jacks-or-better-network`. Optional flags: +`--epochs` (default 20), `--lr`, `--batch-size`, `--hidden-size`, `--seed`. ### 3. Evaluate ```bash python evaluate.py \ - --checkpoint checkpoints/jacks_or_better_network.pt \ + --artifact jacks-or-better-network:latest \ --project video-poker \ --run-name eval-baseline ``` -This plays 100,000 hands with the checkpoint (change with `--hands`), which -takes about 10 seconds, and logs the result as a separate run in the same -project. +This downloads the latest version of the model artifact and plays 100,000 +hands with it (change with `--hands`), which takes about 10 seconds. It logs +to a separate run in the same project. ## What you will see in W&B - **Training run**: config on the Overview tab, and these charts per epoch: - `train_loss` and `val_loss` - `val_optimal_action_pct`: how often the network makes the optimal hold + - `val_mean_regret`: the expected reward per credit bet that the network's + holds give up compared with the optimal holds, on average. Lower is + better, and the saved checkpoint is the epoch with the lowest value. - `val_expected_return_pct`: expected rewards as a percentage of credits wagered - - The checkpoint is under Artifacts. -- **Evaluation run**: in the run summary, `wagered` (credits bet), `payout` - (credits rewarded), `profit` (the difference) and `return_pct` (rewards as - a percentage of credits wagered). +- **Evaluation run**: running totals logged every 100 hands, so you can + watch the score settle as more hands are played: `wagered` (credits bet), + `payout` (credits rewarded), `profit` (the difference) and `return_pct` + (rewards as a percentage of credits wagered). +- **Model artifact**: every training run adds a new version of + `jacks-or-better-network`. Its Lineage tab shows which training run + produced each version and which evaluation runs used it. ## Files @@ -114,10 +121,9 @@ project. | --- | --- | | `generate_dataset.py` | Builds the labeled training dataset. | | `train.py` | Trains the network and logs the run to W&B. | -| `evaluate.py` | Plays hands with a trained checkpoint and logs the score to W&B. | +| `evaluate.py` | Fetches the trained model from W&B, plays hands with it, and logs the score. | +| `dataset.py` | Loads a generated dataset for training. | | `model.py` | The network, hand encoding, training and validation steps, and checkpoint save/load. | | `game.py` | Cards, deck, dealing and drawing. | | `jacks_or_better.py` | Hand rankings and the paytable. | | `ev.py` | Calculates the exact expected reward of each hold choice. | -| `data/dataset.py` | Loads a generated dataset for training. | -| `data/generate.py` | Deals and labels the hands for `generate_dataset.py`. | diff --git a/examples/pytorch/pytorch-video-poker-bot/data/dataset.py b/examples/pytorch/pytorch-video-poker-bot/data/dataset.py deleted file mode 100644 index f036b46c..00000000 --- a/examples/pytorch/pytorch-video-poker-bot/data/dataset.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Load the Jacks or Better training dataset.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - -import numpy as np - -from model import ( - CARD_FEATURES, - CARDS_PER_HAND, - INPUT_SIZE, - NUM_ACTIONS, - RANK_FEATURES, -) - - -@dataclass(frozen=True) -class DatasetMeta: - game_id: str - bet: int - hand_count: int - - -@dataclass(frozen=True) -class HoldDataset: - metadata: DatasetMeta - cards: np.ndarray - targets: np.ndarray - - @property - def states(self) -> np.ndarray: - return encode_cards(self.cards) - - def split_samples( - self, - *, - seed: int, - train_fraction: float = 0.8, - validation_fraction: float = 0.1, - ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - """Shuffle into train / validation / test (80/10/10).""" - rng = np.random.default_rng(seed) - order = rng.permutation(len(self.cards)) - train_end = int(len(order) * train_fraction) - val_end = train_end + int(len(order) * validation_fraction) - return order[:train_end], order[train_end:val_end], order[val_end:] - - -def encode_cards(cards: np.ndarray) -> np.ndarray: - """Batch-encode stored card codes into model inputs.""" - card_codes = np.asarray(cards) - if card_codes.ndim != 2 or card_codes.shape[1] != CARDS_PER_HAND: - raise ValueError("cards must have shape (N, 5)") - ranks = card_codes // 4 - suits = card_codes % 4 - rows = np.arange(len(card_codes))[:, None] - positions = np.arange(CARDS_PER_HAND)[None, :] - encoded = np.zeros((len(card_codes), CARDS_PER_HAND, CARD_FEATURES), dtype=np.float32) - encoded[rows, positions, ranks] = 1.0 - encoded[rows, positions, RANK_FEATURES + suits] = 1.0 - return encoded.reshape(len(card_codes), INPUT_SIZE) - - -def load_dataset(path: Path) -> HoldDataset: - if not path.exists(): - raise FileNotFoundError(f"{path} not found; create it with generate_dataset.py --output {path}") - with np.load(path, allow_pickle=False) as payload: - meta = _read_meta(payload) - cards = payload["cards"] - targets = payload["targets"] - - if cards.shape != (meta.hand_count, CARDS_PER_HAND): - raise ValueError(f"invalid cards shape: {cards.shape}") - if targets.shape != (meta.hand_count, NUM_ACTIONS): - raise ValueError(f"invalid targets shape: {targets.shape}") - - return HoldDataset(metadata=meta, cards=cards, targets=targets) - - -def _read_meta(payload: np.lib.npyio.NpzFile) -> DatasetMeta: - return DatasetMeta( - game_id=str(payload["game"][0]), - bet=int(payload["bet"][0]), - hand_count=int(payload["hand_count"][0]), - ) diff --git a/examples/pytorch/pytorch-video-poker-bot/data/generate.py b/examples/pytorch/pytorch-video-poker-bot/data/generate.py deleted file mode 100644 index ad26822e..00000000 --- a/examples/pytorch/pytorch-video-poker-bot/data/generate.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Build the Jacks or Better training dataset (1M random deals, EV-labeled).""" - -from __future__ import annotations - -import random -import time -from pathlib import Path - -import numpy as np - -from ev import hold_expected_values -from game import BET, card_to_code, make_deck -from jacks_or_better import GAME_ID, PAYTABLE -from model import NUM_ACTIONS, encode_hand - -DEFAULT_HANDS = 1_000_000 -DEFAULT_SEED = 42 - - -def generate_dataset( - path: Path, - *, - hands: int = DEFAULT_HANDS, - seed: int = DEFAULT_SEED, - force: bool = False, -) -> Path: - if path.exists() and not force: - raise FileExistsError(f"refusing to overwrite {path}; pass --force") - - deck = make_deck() - rng = random.Random(seed) - cards = np.empty((hands, 5), dtype=np.uint8) - targets = np.empty((hands, NUM_ACTIONS), dtype=np.float32) - - started = time.perf_counter() - print(f"labeling {hands:,} random deals", flush=True) - for index in range(hands): - encoded = encode_hand(rng.sample(deck, 5)) - cards[index] = [card_to_code(card) for card in encoded.canonical_cards] - values = np.asarray( - hold_expected_values(encoded.canonical_cards, PAYTABLE), - dtype=np.float32, - ) - targets[index] = values / BET - if (index + 1) % 100_000 == 0 or index + 1 == hands: - print(f"labeled {index + 1:,}/{hands:,}", flush=True) - - path.parent.mkdir(parents=True, exist_ok=True) - np.savez_compressed( - path, - cards=cards, - targets=targets, - game=np.asarray([GAME_ID]), - bet=np.asarray([BET], dtype=np.uint8), - hand_count=np.asarray([hands], dtype=np.uint32), - seed=np.asarray([seed], dtype=np.uint64), - ) - print(f"wrote {path} ({path.stat().st_size / 1e6:.1f} MB) in {time.perf_counter() - started:.1f}s") - return path diff --git a/examples/pytorch/pytorch-video-poker-bot/dataset.py b/examples/pytorch/pytorch-video-poker-bot/dataset.py new file mode 100644 index 00000000..7e09d257 --- /dev/null +++ b/examples/pytorch/pytorch-video-poker-bot/dataset.py @@ -0,0 +1,48 @@ +"""Load a dataset written by generate_dataset.py.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +from game import CARDS_PER_HAND, NUM_HOLDS +from model import encode_cards + + +@dataclass(frozen=True) +class HoldDataset: + cards: np.ndarray # (N, 5) card codes, sorted + targets: np.ndarray # (N, 32) expected profit per credit bet for each hold + + @property + def states(self) -> np.ndarray: + return encode_cards(self.cards) + + def split_samples( + self, + *, + seed: int, + validation_fraction: float = 0.1, + ) -> tuple[np.ndarray, np.ndarray]: + """Shuffle into train / validation (90/10).""" + rng = np.random.default_rng(seed) + order = rng.permutation(len(self.cards)) + val_start = len(order) - int(len(order) * validation_fraction) + return order[:val_start], order[val_start:] + + +def load_dataset(path: Path) -> HoldDataset: + if not path.exists(): + raise FileNotFoundError(f"{path} not found; create it with generate_dataset.py --output {path}") + with np.load(path, allow_pickle=False) as payload: + cards = payload["cards"] + targets = payload["targets"] + + if cards.ndim != 2 or cards.shape[1] != CARDS_PER_HAND: + raise ValueError(f"invalid cards shape: {cards.shape}") + if targets.shape != (len(cards), NUM_HOLDS): + raise ValueError(f"invalid targets shape: {targets.shape}") + + return HoldDataset(cards=cards, targets=targets) diff --git a/examples/pytorch/pytorch-video-poker-bot/ev.py b/examples/pytorch/pytorch-video-poker-bot/ev.py index 97802c79..b1fde758 100644 --- a/examples/pytorch/pytorch-video-poker-bot/ev.py +++ b/examples/pytorch/pytorch-video-poker-bot/ev.py @@ -1,7 +1,7 @@ """Exact expected-value calculator for hold decisions. Uses Jacks or Better's vectorized classifier to precompute subset tables, -then answers EV for each of the 32 hold masks with a handful of lookups. +then answers EV for every hold mask with a handful of lookups. """ from __future__ import annotations @@ -12,13 +12,12 @@ import numpy as np -from game import BET, Card, Paytable, hand_to_codes +from game import BET, CARDS_PER_HAND, DECK_SIZE, NUM_HOLDS, Card, Paytable, hand_to_codes from jacks_or_better import NUM_RANK_CLASSES, RANK_CLASSES, classify -CARDS_PER_HAND = 5 -DECK_SIZE = 52 TOTAL_HANDS = comb(DECK_SIZE, CARDS_PER_HAND) -ALL_POSITIONS = 0b11111 +ALL_POSITIONS = NUM_HOLDS - 1 # every card position set +CARDS_LEFT_IN_DECK = DECK_SIZE - CARDS_PER_HAND _BINOMIAL = np.array( [[comb(n, k) for k in range(CARDS_PER_HAND + 1)] for n in range(DECK_SIZE + 1)], @@ -92,24 +91,28 @@ def get_tables() -> EVTables: return _TABLES -def hold_expected_values(hand: Sequence[Card], paytable: Paytable) -> list[float]: - """Exact expected profit for each of the 32 hold masks (BET credits).""" +def rank_payouts(paytable: Paytable) -> np.ndarray: + """Reward for each hand rank, in the order `classify` numbers them.""" + return np.array([paytable.payout_for(rank) for rank in RANK_CLASSES], dtype=np.int64) + + +def hold_expected_values(hand: Sequence[Card], payouts: np.ndarray) -> list[float]: + """Exact expected profit (in credits) for every hold mask. + + `payouts` comes from `rank_payouts`; build it once and reuse it. + """ tables = get_tables() codes = hand_to_codes(hand) - payouts = np.array( - [paytable.payout_for_rank(rank) for rank in RANK_CLASSES], - dtype=np.int64, - ) - subset_payout = [0] * 32 - for subset in range(32): + subset_payout = [0] * NUM_HOLDS + for subset in range(NUM_HOLDS): sorted_codes = sorted( - codes[position] for position in range(5) if subset & (1 << position) + codes[position] for position in range(CARDS_PER_HAND) if subset & (1 << position) ) subset_payout[subset] = int(tables.counts_for(sorted_codes) @ payouts) expected_values: list[float] = [] - for hold_mask in range(32): + for hold_mask in range(NUM_HOLDS): discards = ALL_POSITIONS ^ hold_mask total = 0 subset = discards @@ -119,11 +122,6 @@ def hold_expected_values(hand: Sequence[Card], paytable: Paytable) -> list[float if subset == 0: break subset = (subset - 1) & discards - draws = comb(47, bin(discards).count("1")) + draws = comb(CARDS_LEFT_IN_DECK, bin(discards).count("1")) expected_values.append(total / draws - BET) return expected_values - - -def best_hold_mask(hand: Sequence[Card], paytable: Paytable) -> int: - values = hold_expected_values(hand, paytable) - return int(max(range(32), key=lambda mask: values[mask])) diff --git a/examples/pytorch/pytorch-video-poker-bot/evaluate.py b/examples/pytorch/pytorch-video-poker-bot/evaluate.py index 3ad9d566..4532575e 100644 --- a/examples/pytorch/pytorch-video-poker-bot/evaluate.py +++ b/examples/pytorch/pytorch-video-poker-bot/evaluate.py @@ -1,54 +1,72 @@ #!/usr/bin/env python3 -"""Evaluate a trained hold-network checkpoint with greedy play.""" +"""Fetch a trained network from W&B and play hands with it, logging as it goes.""" from __future__ import annotations import argparse +import random from pathlib import Path import wandb +from game import BET, CARDS_PER_HAND, VideoPokerGame, deal, make_deck, shuffle_deck from jacks_or_better import make_game -from model import load_checkpoint, play_hands +from model import Network, choose_hold, load_checkpoint + + +def play_hands( + run: wandb.Run, + model: Network, + game: VideoPokerGame, + *, + hands: int, + seed: int, + log_every: int = 100, +) -> None: + """Play greedy hands, logging and printing running totals every `log_every` hands.""" + rng = random.Random(seed) + wagered = 0 + payout = 0 + for hand_num in range(1, hands + 1): + deck = make_deck() + shuffle_deck(deck, rng=rng) + dealt = deal(deck, CARDS_PER_HAND) + payout += game.play_hand(deck, dealt, choose_hold(model, dealt)) + wagered += BET + if hand_num % log_every == 0 or hand_num == hands: + return_pct = (payout / wagered) * 100 + run.log( + { + "hands_played": hand_num, + "wagered": wagered, + "payout": payout, + "profit": payout - wagered, + "return_pct": return_pct, + }, + step=hand_num, + ) + print(f"hand {hand_num:,}/{hands:,} return={return_pct:.2f}% profit={payout - wagered:,}", flush=True) def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--checkpoint", type=Path, required=True, help=".pt saved by train.py") + parser.add_argument("--artifact", required=True, help="model artifact from train.py, e.g. my-model:latest") parser.add_argument("--project", required=True, help="W&B project to log to") parser.add_argument("--run-name", required=True, help="name for this W&B run") parser.add_argument("--hands", type=int, default=100_000) parser.add_argument("--seed", type=int, default=42) args = parser.parse_args() - if not args.checkpoint.exists(): - raise SystemExit(f"checkpoint not found: {args.checkpoint}") - - model, config = load_checkpoint(args.checkpoint) - game = make_game() - - print(f"checkpoint={args.checkpoint} hands={args.hands:,}", flush=True) - with wandb.init( project=args.project, name=args.run_name, job_type="evaluation", - config={ - "checkpoint": str(args.checkpoint), - "hands": args.hands, - "seed": args.seed, - **config, - }, + config={"artifact": args.artifact, "hands": args.hands, "seed": args.seed}, ) as run: - metrics = play_hands( - model=model, - game=game, - hands=args.hands, - seed=args.seed, - ) - run.summary.update(metrics) - run.log(metrics) - print(f"return_pct={metrics['return_pct']:.2f}% profit={int(metrics['profit']):,}") + # use_artifact records that this run consumed the model, linking it to the training run. + checkpoint = Path(run.use_artifact(args.artifact).file()) + model = load_checkpoint(checkpoint) + play_hands(run, model, make_game(), hands=args.hands, seed=args.seed) if __name__ == "__main__": diff --git a/examples/pytorch/pytorch-video-poker-bot/game.py b/examples/pytorch/pytorch-video-poker-bot/game.py index bf55a5ff..6af4faa9 100644 --- a/examples/pytorch/pytorch-video-poker-bot/game.py +++ b/examples/pytorch/pytorch-video-poker-bot/game.py @@ -1,6 +1,6 @@ """Video poker core: cards, ranks, paytable, and the deal/hold/draw loop. -The bet is always five coins — baked into BET below. +The bet is always five credits, baked into BET below. """ from __future__ import annotations @@ -10,7 +10,10 @@ from enum import Enum from typing import Callable, Mapping, Sequence -BET = 5 # always max-coin Jacks or Better +BET = 5 # credits bet per hand +CARDS_PER_HAND = 5 +DECK_SIZE = 52 +NUM_HOLDS = 1 << CARDS_PER_HAND # 32 ways to choose which cards to hold RANKS = tuple(range(2, 15)) # 2..14, Ace high SUITS = (0, 1, 2, 3) @@ -34,30 +37,6 @@ def __str__(self) -> str: return f"{rank}{SUIT_CHARS[self.suit]}" -def parse_card(text: str) -> Card: - """Parse strings like 'Ah', 'Td', '7s'.""" - text = text.strip() - if len(text) != 2: - raise ValueError(f"invalid card: {text}") - rank_char, suit_char = text[0].upper(), text[1].lower() - rank_map = {"T": 10, **{v: k for k, v in RANK_CHARS.items()}} - if rank_char.isdigit(): - rank = int(rank_char) - else: - rank = rank_map.get(rank_char) - if rank is None or rank not in RANKS: - raise ValueError(f"invalid rank in card: {text}") - try: - suit = SUIT_CHARS.index(suit_char) - except ValueError as exc: - raise ValueError(f"invalid suit in card: {text}") from exc - return Card(rank=rank, suit=suit) - - -def cards_from_strings(values: Sequence[str]) -> list[Card]: - return [parse_card(value) for value in values] - - def make_deck() -> list[Card]: return [Card(rank=rank, suit=suit) for suit in SUITS for rank in RANKS] @@ -67,10 +46,6 @@ def card_to_code(card: Card) -> int: return (card.rank - 2) * 4 + card.suit -def code_to_card(code: int) -> Card: - return Card(rank=(code // 4) + 2, suit=code % 4) - - def hand_to_codes(hand: Sequence[Card]) -> list[int]: return [card_to_code(card) for card in hand] @@ -88,10 +63,10 @@ def deal(deck: list[Card], count: int) -> list[Card]: def apply_hold(hand: Sequence[Card], hold_mask: int, deck: list[Card]) -> list[Card]: - if hold_mask < 0 or hold_mask > 31: - raise ValueError("hold_mask must be a 5-bit integer (0-31)") + if not 0 <= hold_mask < NUM_HOLDS: + raise ValueError(f"hold_mask must be 0-{NUM_HOLDS - 1}") held = [card for index, card in enumerate(hand) if hold_mask & (1 << index)] - draw_count = 5 - len(held) + draw_count = CARDS_PER_HAND - len(held) return held + (deal(deck, draw_count) if draw_count else []) @@ -112,70 +87,25 @@ class HandRank(Enum): class EvaluatedHand: rank: HandRank - def __str__(self) -> str: - return self.rank.value - @dataclass(frozen=True) class Paytable: """Maps a hand rank to total credits paid (for a fixed BET).""" - name: str payouts: Mapping[HandRank, int] = field(default_factory=dict) - def payout_for(self, hand: EvaluatedHand) -> int: - return self.payouts.get(hand.rank, 0) - - def payout_for_rank(self, rank: HandRank) -> int: + def payout_for(self, rank: HandRank) -> int: return self.payouts.get(rank, 0) -@dataclass(frozen=True) -class PlayResult: - initial_hand: tuple[Card, ...] - final_hand: tuple[Card, ...] - evaluated: EvaluatedHand - payout: int - profit: int - - @dataclass class VideoPokerGame: paytable: Paytable evaluate: Callable[[Sequence[Card]], EvaluatedHand] - def evaluate_hand(self, cards: Sequence[Card]) -> EvaluatedHand: - return self.evaluate(cards) - - def payout_for_hand(self, cards: Sequence[Card]) -> int: - return self.paytable.payout_for(self.evaluate_hand(cards)) - - def play_hand( - self, - hold_mask: int, - rng: random.Random | None = None, - ) -> PlayResult: - rng = rng or random - deck = make_deck() - shuffle_deck(deck, rng=rng) - return self.play_hand_with_state(deck, deal(deck, 5), hold_mask) - - def play_hand_with_state( - self, - deck: list[Card], - hand: Sequence[Card], - hold_mask: int, - ) -> PlayResult: - if len(hand) != 5: + def play_hand(self, deck: list[Card], hand: Sequence[Card], hold_mask: int) -> int: + """Hold, draw from `deck`, and return the reward in credits.""" + if len(hand) != CARDS_PER_HAND: raise ValueError("expected a 5-card hand") - initial = tuple(hand) - final = tuple(apply_hold(initial, hold_mask, deck)) - evaluated = self.evaluate_hand(final) - payout = self.paytable.payout_for(evaluated) - return PlayResult( - initial_hand=initial, - final_hand=final, - evaluated=evaluated, - payout=payout, - profit=payout - BET, - ) + final = apply_hold(hand, hold_mask, deck) + return self.paytable.payout_for(self.evaluate(final).rank) diff --git a/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py b/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py index 1418e4e7..f222d599 100644 --- a/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py +++ b/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py @@ -1,27 +1,50 @@ #!/usr/bin/env python3 -"""Generate the Jacks or Better training dataset labeled with exact hold EVs.""" +"""Generate the training dataset: random deals labeled with the exact expected +reward of every hold.""" from __future__ import annotations import argparse +import random +import time from pathlib import Path -from data.generate import DEFAULT_HANDS, DEFAULT_SEED, generate_dataset +import numpy as np + +from ev import hold_expected_values, rank_payouts +from game import BET, CARDS_PER_HAND, NUM_HOLDS, card_to_code, make_deck +from jacks_or_better import PAYTABLE + + +def generate_dataset(path: Path, *, hands: int, seed: int) -> None: + deck = make_deck() + rng = random.Random(seed) + payouts = rank_payouts(PAYTABLE) + cards = np.empty((hands, CARDS_PER_HAND), dtype=np.uint8) + targets = np.empty((hands, NUM_HOLDS), dtype=np.float32) + + started = time.perf_counter() + print(f"labeling {hands:,} random deals", flush=True) + for index in range(hands): + # Sorted by card code: the same order model.choose_hold feeds the network. + hand = sorted(rng.sample(deck, CARDS_PER_HAND), key=card_to_code) + cards[index] = [card_to_code(card) for card in hand] + targets[index] = np.asarray(hold_expected_values(hand, payouts)) / BET + if (index + 1) % 100_000 == 0 or index + 1 == hands: + print(f"labeled {index + 1:,}/{hands:,}", flush=True) + + path.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed(path, cards=cards, targets=targets) + print(f"wrote {path} ({path.stat().st_size / 1e6:.1f} MB) in {time.perf_counter() - started:.1f}s") def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", type=Path, required=True, help="where to write the dataset (.npz)") - parser.add_argument("--hands", type=int, default=DEFAULT_HANDS) - parser.add_argument("--seed", type=int, default=DEFAULT_SEED) - parser.add_argument("--force", action="store_true", help="overwrite existing file") + parser.add_argument("--hands", type=int, default=1_000_000) + parser.add_argument("--seed", type=int, default=42) args = parser.parse_args() - generate_dataset( - args.output, - hands=args.hands, - seed=args.seed, - force=args.force, - ) + generate_dataset(args.output, hands=args.hands, seed=args.seed) if __name__ == "__main__": diff --git a/examples/pytorch/pytorch-video-poker-bot/jacks_or_better.py b/examples/pytorch/pytorch-video-poker-bot/jacks_or_better.py index 5bedea49..639dda64 100644 --- a/examples/pytorch/pytorch-video-poker-bot/jacks_or_better.py +++ b/examples/pytorch/pytorch-video-poker-bot/jacks_or_better.py @@ -11,6 +11,7 @@ import numpy as np from game import ( + CARDS_PER_HAND, Card, EvaluatedHand, HandRank, @@ -19,11 +20,8 @@ hand_to_codes, ) -GAME_ID = "jacks_or_better_9_6" - # Absolute credits paid at BET=5 (royal includes the max-coin bonus). PAYTABLE = Paytable( - name="9/6 Jacks or Better", payouts={ HandRank.ROYAL_FLUSH: 4000, HandRank.STRAIGHT_FLUSH: 250, @@ -61,7 +59,7 @@ def classify(hands: np.ndarray) -> np.ndarray: """Rank many hands at once. Each row is five card codes; result is RANK_CLASSES index.""" codes = np.asarray(hands) - if codes.ndim != 2 or codes.shape[1] != 5: + if codes.ndim != 2 or codes.shape[1] != CARDS_PER_HAND: raise ValueError("expected an (N, 5) array of card codes") codes = np.sort(codes, axis=1) @@ -106,7 +104,7 @@ def classify(hands: np.ndarray) -> np.ndarray: def evaluate_hand(cards: Sequence[Card]) -> EvaluatedHand: """Classify one hand; translate the fast ordinal into EvaluatedHand.""" - if len(cards) != 5: + if len(cards) != CARDS_PER_HAND: raise ValueError("expected exactly 5 cards") ordinal = int(classify(np.asarray([hand_to_codes(cards)], dtype=np.uint8))[0]) return EvaluatedHand(RANK_CLASSES[ordinal]) diff --git a/examples/pytorch/pytorch-video-poker-bot/model.py b/examples/pytorch/pytorch-video-poker-bot/model.py index 248e9b7c..de9f2e75 100644 --- a/examples/pytorch/pytorch-video-poker-bot/model.py +++ b/examples/pytorch/pytorch-video-poker-bot/model.py @@ -1,81 +1,45 @@ -"""Hold network: encode a hand, score the 32 holds, train, save, and play. +"""The network: encode hands, score the 32 holds, train, validate, save and load. -A checkpoint is the learned weights (plus a small config so we can rebuild -the network). Everything runs on CPU — the model is tiny. +Everything runs on CPU; the model is tiny. """ from __future__ import annotations -import random -from dataclasses import dataclass from pathlib import Path -from typing import Any, Sequence +from typing import Sequence import numpy as np import torch from torch import nn from torch.nn import functional as F -from game import BET, Card, VideoPokerGame, deal, make_deck, shuffle_deck +from game import CARDS_PER_HAND, NUM_HOLDS, Card, card_to_code -CARDS_PER_HAND = 5 RANK_FEATURES = 13 SUIT_FEATURES = 4 CARD_FEATURES = RANK_FEATURES + SUIT_FEATURES INPUT_SIZE = CARDS_PER_HAND * CARD_FEATURES -NUM_ACTIONS = 1 << CARDS_PER_HAND # 32 hold patterns - - -@dataclass(frozen=True) -class EncodedHand: - values: np.ndarray - canonical_cards: tuple[Card, ...] - - def hold_mask(self, action: int, dealt_hand: Sequence[Card]) -> int: - """Map a canonical action index back onto the physical dealt order.""" - if action < 0 or action >= NUM_ACTIONS: - raise ValueError(f"action must be 0-{NUM_ACTIONS - 1}") - held = { - card - for index, card in enumerate(self.canonical_cards) - if action & (1 << index) - } - mask = 0 - for index, card in enumerate(dealt_hand): - if card in held: - mask |= 1 << index - if len(held) != bin(mask).count("1"): - raise ValueError("canonical cards must all be present in dealt hand") - return mask - - -def encode_hand(hand: Sequence[Card]) -> EncodedHand: - """Encode a hand invariant to deal order and physical suit names.""" - if len(hand) != CARDS_PER_HAND: - raise ValueError("expected exactly 5 cards") - if len(set(hand)) != CARDS_PER_HAND: - raise ValueError("hand contains duplicate cards") - - suit_rank_masks = [0] * SUIT_FEATURES - for card in hand: - suit_rank_masks[card.suit] |= 1 << (card.rank - 2) - suit_order = sorted(range(SUIT_FEATURES), key=lambda s: (-suit_rank_masks[s], s)) - suit_map = [0] * SUIT_FEATURES - for canonical_suit, physical_suit in enumerate(suit_order): - suit_map[physical_suit] = canonical_suit - - best_cards = tuple(sorted(hand, key=lambda c: (-c.rank, suit_map[c.suit]))) - values = np.zeros((CARDS_PER_HAND, CARD_FEATURES), dtype=np.float32) - for index, card in enumerate(best_cards): - values[index, card.rank - 2] = 1.0 - values[index, RANK_FEATURES + suit_map[card.suit]] = 1.0 - return EncodedHand(values=values.reshape(INPUT_SIZE), canonical_cards=best_cards) - - -class HoldNetwork(nn.Module): + + +def encode_cards(cards: np.ndarray) -> np.ndarray: + """One-hot encode (N, 5) card codes into (N, INPUT_SIZE) network inputs.""" + card_codes = np.asarray(cards) + if card_codes.ndim != 2 or card_codes.shape[1] != CARDS_PER_HAND: + raise ValueError("cards must have shape (N, 5)") + ranks = card_codes // 4 + suits = card_codes % 4 + rows = np.arange(len(card_codes))[:, None] + positions = np.arange(CARDS_PER_HAND)[None, :] + encoded = np.zeros((len(card_codes), CARDS_PER_HAND, CARD_FEATURES), dtype=np.float32) + encoded[rows, positions, ranks] = 1.0 + encoded[rows, positions, RANK_FEATURES + suits] = 1.0 + return encoded.reshape(len(card_codes), INPUT_SIZE) + + +class Network(nn.Module): """MLP that scores each of the 32 possible hold patterns.""" - def __init__(self, hidden_size: int = 256) -> None: + def __init__(self, hidden_size: int) -> None: super().__init__() self.hidden_size = hidden_size self.layers = nn.Sequential( @@ -83,76 +47,40 @@ def __init__(self, hidden_size: int = 256) -> None: nn.ReLU(), nn.Linear(hidden_size, hidden_size), nn.ReLU(), - nn.Linear(hidden_size, NUM_ACTIONS), + nn.Linear(hidden_size, NUM_HOLDS), ) def forward(self, inputs: torch.Tensor) -> torch.Tensor: return self.layers(inputs) -def save_checkpoint(path: Path, *, model: HoldNetwork, config: dict[str, Any]) -> None: +def save_checkpoint(path: Path, model: Network) -> None: path.parent.mkdir(parents=True, exist_ok=True) - torch.save({"weights": model.state_dict(), "config": config}, path) + torch.save({"hidden_size": model.hidden_size, "weights": model.state_dict()}, path) -def load_checkpoint(path: Path) -> tuple[HoldNetwork, dict[str, Any]]: - payload = torch.load(path, map_location="cpu", weights_only=False) - config = dict(payload.get("config", {})) - # Older checkpoints used "model_state"; prefer "weights". - state = payload.get("weights") or payload["model_state"] - model = HoldNetwork(hidden_size=int(config.get("hidden_size", 256))) - model.load_state_dict(state) +def load_checkpoint(path: Path) -> Network: + payload = torch.load(path, map_location="cpu", weights_only=True) + model = Network(payload["hidden_size"]) + model.load_state_dict(payload["weights"]) model.eval() - return model, config - + return model -def choose_hold(model: HoldNetwork, hand: Sequence[Card]) -> int: - """Pick the highest-scoring hold pattern for this hand.""" - encoded = encode_hand(hand) - with torch.no_grad(): - scores = model(torch.from_numpy(encoded.values).unsqueeze(0)) - action = int(scores.argmax(dim=1).item()) - return encoded.hold_mask(action, hand) +def choose_hold(model: Network, hand: Sequence[Card]) -> int: + """Pick the highest-scoring hold; returns a hold mask over `hand` as dealt. -def play_hands( - *, - model: HoldNetwork, - game: VideoPokerGame, - hands: int, - seed: int = 42, - log_every: int = 10_000, -) -> dict[str, float]: - """Play greedy hands; return return_pct / profit / wagered / payout.""" - rng = random.Random(seed) - wagered = 0 - payout = 0 - for hand_num in range(1, hands + 1): - deck = make_deck() - shuffle_deck(deck, rng=rng) - dealt = deal(deck, 5) - result = game.play_hand_with_state(deck, dealt, choose_hold(model, dealt)) - wagered += BET - payout += result.payout - if hand_num % log_every == 0 or hand_num == hands: - print( - f"hand {hand_num:,}/{hands:,} " - f"return={(payout / wagered) * 100:.2f}% " - f"profit={payout - wagered:,}", - flush=True, - ) - return { - "hands": float(hands), - "bet": float(BET), - "wagered": float(wagered), - "payout": float(payout), - "profit": float(payout - wagered), - "return_pct": (payout / wagered) * 100, - } + The network sees cards sorted by card code, the same order the dataset uses. + """ + order = sorted(range(CARDS_PER_HAND), key=lambda i: card_to_code(hand[i])) + codes = np.array([[card_to_code(hand[i]) for i in order]]) + with torch.no_grad(): + action = int(model(torch.from_numpy(encode_cards(codes))).argmax(dim=1).item()) + return sum(1 << order[bit] for bit in range(CARDS_PER_HAND) if action & (1 << bit)) def train_epoch( - model: HoldNetwork, + model: Network, optimizer: torch.optim.Optimizer, *, states: np.ndarray, @@ -182,14 +110,14 @@ def train_epoch( @torch.no_grad() def validation_metrics( - model: HoldNetwork, + model: Network, *, states: np.ndarray, targets: np.ndarray, sample_ids: np.ndarray, batch_size: int, ) -> dict[str, float]: - """Loss, % optimal action, and expected return vs the EV labels.""" + """Loss, % optimal holds, mean regret, and expected return vs the EV labels.""" model.eval() total_loss = total_regret = total_policy = 0.0 optimal = n = 0 @@ -213,6 +141,6 @@ def validation_metrics( return { "loss": total_loss / (n * targets.shape[1]), "optimal_action_pct": (optimal / n) * 100, - "mean_ev_regret": total_regret / n, + "mean_regret": total_regret / n, "expected_return_pct": (1.0 + total_policy / n) * 100, } diff --git a/examples/pytorch/pytorch-video-poker-bot/train.py b/examples/pytorch/pytorch-video-poker-bot/train.py index ed383320..399cce0a 100644 --- a/examples/pytorch/pytorch-video-poker-bot/train.py +++ b/examples/pytorch/pytorch-video-poker-bot/train.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Train a hold network on exact EV targets and log the run to W&B.""" +"""Train the network on exact expected-reward targets and log the run to W&B.""" from __future__ import annotations @@ -10,22 +10,15 @@ import torch import wandb -from data.dataset import load_dataset -from game import BET -from model import ( - INPUT_SIZE, - NUM_ACTIONS, - HoldNetwork, - save_checkpoint, - train_epoch, - validation_metrics, -) +from dataset import load_dataset +from model import Network, save_checkpoint, train_epoch, validation_metrics def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--dataset", type=Path, required=True, help=".npz from generate_dataset.py") parser.add_argument("--checkpoint", type=Path, required=True, help="where to save the trained model (.pt)") + parser.add_argument("--artifact-name", required=True, help="W&B model artifact to upload the checkpoint as") parser.add_argument("--project", required=True, help="W&B project to log to") parser.add_argument("--run-name", required=True, help="name for this W&B run") parser.add_argument("--epochs", type=int, default=20) @@ -36,12 +29,11 @@ def main() -> None: args = parser.parse_args() torch.manual_seed(args.seed) - np.random.seed(args.seed) data = load_dataset(args.dataset) states = data.states - train_ids, val_ids, _test_ids = data.split_samples(seed=args.seed) - model = HoldNetwork(args.hidden_size) + train_ids, val_ids = data.split_samples(seed=args.seed) + model = Network(args.hidden_size) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) rng = np.random.default_rng(args.seed) @@ -51,19 +43,11 @@ def main() -> None: "lr": args.lr, "hidden_size": args.hidden_size, "seed": args.seed, - "bet": BET, - "input_size": INPUT_SIZE, - "num_actions": NUM_ACTIONS, - "game": data.metadata.game_id, - "hands": data.metadata.hand_count, + "hands": len(data.cards), } # One place for init → log → finish (finish runs automatically on exit). - with wandb.init( - project=args.project, - name=args.run_name, - config=config, - ) as run: + with wandb.init(project=args.project, name=args.run_name, config=config) as run: best_regret = float("inf") for epoch in range(1, args.epochs + 1): train_loss = train_epoch( @@ -88,6 +72,7 @@ def main() -> None: "train_loss": train_loss, "val_loss": val["loss"], "val_optimal_action_pct": val["optimal_action_pct"], + "val_mean_regret": val["mean_regret"], "val_expected_return_pct": val["expected_return_pct"], }, step=epoch, @@ -96,16 +81,17 @@ def main() -> None: f"epoch {epoch}/{args.epochs} " f"loss={train_loss:.4f} " f"val_optimal={val['optimal_action_pct']:.1f}% " + f"val_regret={val['mean_regret']:.4f} " f"val_return={val['expected_return_pct']:.2f}%" ) - if val["mean_ev_regret"] < best_regret: - best_regret = val["mean_ev_regret"] - save_checkpoint(args.checkpoint, model=model, config=config) + if val["mean_regret"] < best_regret: + best_regret = val["mean_regret"] + save_checkpoint(args.checkpoint, model) - artifact = wandb.Artifact(args.checkpoint.stem, type="model") + artifact = wandb.Artifact(args.artifact_name, type="model") artifact.add_file(str(args.checkpoint)) run.log_artifact(artifact) - print(f"saved {args.checkpoint}") + print(f"saved {args.checkpoint} and logged artifact {args.artifact_name}") if __name__ == "__main__": From 02fdcc1f623d3b07694b7ae95a86dd4ffc0a8b58 Mon Sep 17 00:00:00 2001 From: Jonathan Stack Date: Tue, 22 Sep 2026 21:03:46 -0700 Subject: [PATCH 4/6] docs(experiments): randomize video poker seeds and group runs in W&B - train.py and evaluate.py pick a random seed unless --seed is passed. The seed used is recorded in the run config, so any run can be replayed exactly. - Training runs go in the "train" group and evaluation runs in the "eval" group. - README notes both. --- examples/pytorch/pytorch-video-poker-bot/README.md | 11 ++++++++--- examples/pytorch/pytorch-video-poker-bot/evaluate.py | 5 ++++- examples/pytorch/pytorch-video-poker-bot/train.py | 7 +++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/examples/pytorch/pytorch-video-poker-bot/README.md b/examples/pytorch/pytorch-video-poker-bot/README.md index c1c05ea6..b376d80a 100644 --- a/examples/pytorch/pytorch-video-poker-bot/README.md +++ b/examples/pytorch/pytorch-video-poker-bot/README.md @@ -4,8 +4,8 @@ This example trains a small PyTorch network to play video poker and tracks it with [Weights & Biases](https://wandb.ai). The poker is just a stand-in task. The point is to see a complete W&B run: -1. **`wandb.init()`** starts a run and records its config (hyperparameters - and dataset size). +1. **`wandb.init()`** starts a run, puts it in a group (`train` or `eval`), + and records its config (hyperparameters and dataset size). 2. **`run.log()`** sends metrics every epoch, so you can watch training live. 3. **`run.log_artifact()`** uploads the trained checkpoint as a versioned model artifact. @@ -84,6 +84,10 @@ checkpoint if validation improved, and at the end it uploads the checkpoint to W&B as the model artifact `jacks-or-better-network`. Optional flags: `--epochs` (default 20), `--lr`, `--batch-size`, `--hidden-size`, `--seed`. +Without `--seed`, each run picks a random seed. The seed it used is saved +in the run's config in W&B, so you can repeat any run exactly by passing +that value. + ### 3. Evaluate ```bash @@ -95,7 +99,8 @@ python evaluate.py \ This downloads the latest version of the model artifact and plays 100,000 hands with it (change with `--hands`), which takes about 10 seconds. It logs -to a separate run in the same project. +to a separate run in the same project. Like training, it deals with a random +seed unless you pass `--seed`, so each evaluation plays different hands. ## What you will see in W&B diff --git a/examples/pytorch/pytorch-video-poker-bot/evaluate.py b/examples/pytorch/pytorch-video-poker-bot/evaluate.py index 4532575e..2acd6979 100644 --- a/examples/pytorch/pytorch-video-poker-bot/evaluate.py +++ b/examples/pytorch/pytorch-video-poker-bot/evaluate.py @@ -54,13 +54,16 @@ def main() -> None: parser.add_argument("--project", required=True, help="W&B project to log to") parser.add_argument("--run-name", required=True, help="name for this W&B run") parser.add_argument("--hands", type=int, default=100_000) - parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--seed", type=int, help="default: random; the seed used is logged to W&B") args = parser.parse_args() + if args.seed is None: + args.seed = random.randrange(2**32) with wandb.init( project=args.project, name=args.run_name, job_type="evaluation", + group="eval", config={"artifact": args.artifact, "hands": args.hands, "seed": args.seed}, ) as run: # use_artifact records that this run consumed the model, linking it to the training run. diff --git a/examples/pytorch/pytorch-video-poker-bot/train.py b/examples/pytorch/pytorch-video-poker-bot/train.py index 399cce0a..7305ed84 100644 --- a/examples/pytorch/pytorch-video-poker-bot/train.py +++ b/examples/pytorch/pytorch-video-poker-bot/train.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import random from pathlib import Path import numpy as np @@ -25,8 +26,10 @@ def main() -> None: parser.add_argument("--batch-size", type=int, default=1024) parser.add_argument("--lr", type=float, default=1e-3) parser.add_argument("--hidden-size", type=int, default=256) - parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--seed", type=int, help="default: random; the seed used is logged to W&B") args = parser.parse_args() + if args.seed is None: + args.seed = random.randrange(2**32) torch.manual_seed(args.seed) @@ -47,7 +50,7 @@ def main() -> None: } # One place for init → log → finish (finish runs automatically on exit). - with wandb.init(project=args.project, name=args.run_name, config=config) as run: + with wandb.init(project=args.project, name=args.run_name, group="train", config=config) as run: best_regret = float("inf") for epoch in range(1, args.epochs + 1): train_loss = train_epoch( From 341a2baae2b8f93a8d3e397ff82a1c47316db574 Mon Sep 17 00:00:00 2001 From: Jonathan Stack Date: Tue, 22 Sep 2026 21:18:55 -0700 Subject: [PATCH 5/6] docs(experiments): correct video poker file header docstrings - ev.py: describe what it computes (expected profit, not "expected value") and how, including the inclusion-exclusion step that removes draws reusing discarded cards. - game.py: no deal/hold/draw loop and no paytable values live here. - jacks_or_better.py: EvaluatedHand no longer carries anything richer than the rank. - generate_dataset.py, train.py: the targets are expected profit per credit bet, not expected reward. --- examples/pytorch/pytorch-video-poker-bot/ev.py | 13 ++++++++++--- examples/pytorch/pytorch-video-poker-bot/game.py | 4 +++- .../pytorch-video-poker-bot/generate_dataset.py | 2 +- .../pytorch-video-poker-bot/jacks_or_better.py | 2 +- examples/pytorch/pytorch-video-poker-bot/train.py | 2 +- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/examples/pytorch/pytorch-video-poker-bot/ev.py b/examples/pytorch/pytorch-video-poker-bot/ev.py index b1fde758..eeabbe56 100644 --- a/examples/pytorch/pytorch-video-poker-bot/ev.py +++ b/examples/pytorch/pytorch-video-poker-bot/ev.py @@ -1,7 +1,14 @@ -"""Exact expected-value calculator for hold decisions. +"""Exact expected profit of every hold choice, found by counting, not simulating. -Uses Jacks or Better's vectorized classifier to precompute subset tables, -then answers EV for every hold mask with a handful of lookups. +Once per process, `build_tables` ranks all 2,598,960 five-card hands with the +Jacks or Better classifier. For every group of 0 to 4 cards, it counts how many +five-card hands containing that group land in each hand rank. + +For a dealt hand, `hold_expected_values` looks up those counts for the hand's +32 subsets of cards. The counts include hands that reuse discarded cards, which +can't be drawn again, so inclusion-exclusion over the discards removes them. +Averaging the rewards over every possible draw and subtracting the bet gives +the expected profit of each hold. """ from __future__ import annotations diff --git a/examples/pytorch/pytorch-video-poker-bot/game.py b/examples/pytorch/pytorch-video-poker-bot/game.py index 6af4faa9..6b0580b9 100644 --- a/examples/pytorch/pytorch-video-poker-bot/game.py +++ b/examples/pytorch/pytorch-video-poker-bot/game.py @@ -1,4 +1,6 @@ -"""Video poker core: cards, ranks, paytable, and the deal/hold/draw loop. +"""Video poker basics: cards, the deck, dealing and drawing, hand ranks, and +the paytable type. The Jacks or Better rules and rewards are in +jacks_or_better.py. The bet is always five credits, baked into BET below. """ diff --git a/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py b/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py index f222d599..36e5c972 100644 --- a/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py +++ b/examples/pytorch/pytorch-video-poker-bot/generate_dataset.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """Generate the training dataset: random deals labeled with the exact expected -reward of every hold.""" +profit of every hold, per credit bet.""" from __future__ import annotations diff --git a/examples/pytorch/pytorch-video-poker-bot/jacks_or_better.py b/examples/pytorch/pytorch-video-poker-bot/jacks_or_better.py index 639dda64..575872f9 100644 --- a/examples/pytorch/pytorch-video-poker-bot/jacks_or_better.py +++ b/examples/pytorch/pytorch-video-poker-bot/jacks_or_better.py @@ -1,7 +1,7 @@ """9/6 Jacks or Better: ranking, paytable, and game factory. `classify` is the single source of truth for hand ranks (vectorized). -`evaluate_hand` is a thin wrapper that returns the richer EvaluatedHand type. +`evaluate_hand` uses it to rank a single hand. """ from __future__ import annotations diff --git a/examples/pytorch/pytorch-video-poker-bot/train.py b/examples/pytorch/pytorch-video-poker-bot/train.py index 7305ed84..07d36882 100644 --- a/examples/pytorch/pytorch-video-poker-bot/train.py +++ b/examples/pytorch/pytorch-video-poker-bot/train.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Train the network on exact expected-reward targets and log the run to W&B.""" +"""Train the network to predict each hold's exact expected profit and log the run to W&B.""" from __future__ import annotations From 4af823f5e2cf1e9761a4f6f2807824dbfa364f72 Mon Sep 17 00:00:00 2001 From: Jonathan Stack Date: Tue, 22 Sep 2026 21:32:58 -0700 Subject: [PATCH 6/6] docs(experiments): add optional --entity flag to video poker scripts Lets runs log to a W&B team instead of relying on the machine's default entity. Leaving it off keeps the old behavior. --- examples/pytorch/pytorch-video-poker-bot/README.md | 2 ++ examples/pytorch/pytorch-video-poker-bot/evaluate.py | 2 ++ examples/pytorch/pytorch-video-poker-bot/train.py | 9 ++++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/examples/pytorch/pytorch-video-poker-bot/README.md b/examples/pytorch/pytorch-video-poker-bot/README.md index b376d80a..7c65a0a5 100644 --- a/examples/pytorch/pytorch-video-poker-bot/README.md +++ b/examples/pytorch/pytorch-video-poker-bot/README.md @@ -82,6 +82,7 @@ python train.py \ This takes under a minute on a laptop CPU. After each epoch it saves the checkpoint if validation improved, and at the end it uploads the checkpoint to W&B as the model artifact `jacks-or-better-network`. Optional flags: +`--entity` (a W&B team or user; defaults to your default entity), `--epochs` (default 20), `--lr`, `--batch-size`, `--hidden-size`, `--seed`. Without `--seed`, each run picks a random seed. The seed it used is saved @@ -101,6 +102,7 @@ This downloads the latest version of the model artifact and plays 100,000 hands with it (change with `--hands`), which takes about 10 seconds. It logs to a separate run in the same project. Like training, it deals with a random seed unless you pass `--seed`, so each evaluation plays different hands. +If you trained under a team with `--entity`, pass the same `--entity` here. ## What you will see in W&B diff --git a/examples/pytorch/pytorch-video-poker-bot/evaluate.py b/examples/pytorch/pytorch-video-poker-bot/evaluate.py index 2acd6979..d87bc2f1 100644 --- a/examples/pytorch/pytorch-video-poker-bot/evaluate.py +++ b/examples/pytorch/pytorch-video-poker-bot/evaluate.py @@ -51,6 +51,7 @@ def play_hands( def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--artifact", required=True, help="model artifact from train.py, e.g. my-model:latest") + parser.add_argument("--entity", help="W&B team or user to log to (default: your default entity)") parser.add_argument("--project", required=True, help="W&B project to log to") parser.add_argument("--run-name", required=True, help="name for this W&B run") parser.add_argument("--hands", type=int, default=100_000) @@ -60,6 +61,7 @@ def main() -> None: args.seed = random.randrange(2**32) with wandb.init( + entity=args.entity, project=args.project, name=args.run_name, job_type="evaluation", diff --git a/examples/pytorch/pytorch-video-poker-bot/train.py b/examples/pytorch/pytorch-video-poker-bot/train.py index 07d36882..090a0fa2 100644 --- a/examples/pytorch/pytorch-video-poker-bot/train.py +++ b/examples/pytorch/pytorch-video-poker-bot/train.py @@ -20,6 +20,7 @@ def main() -> None: parser.add_argument("--dataset", type=Path, required=True, help=".npz from generate_dataset.py") parser.add_argument("--checkpoint", type=Path, required=True, help="where to save the trained model (.pt)") parser.add_argument("--artifact-name", required=True, help="W&B model artifact to upload the checkpoint as") + parser.add_argument("--entity", help="W&B team or user to log to (default: your default entity)") parser.add_argument("--project", required=True, help="W&B project to log to") parser.add_argument("--run-name", required=True, help="name for this W&B run") parser.add_argument("--epochs", type=int, default=20) @@ -50,7 +51,13 @@ def main() -> None: } # One place for init → log → finish (finish runs automatically on exit). - with wandb.init(project=args.project, name=args.run_name, group="train", config=config) as run: + with wandb.init( + entity=args.entity, + project=args.project, + name=args.run_name, + group="train", + config=config, + ) as run: best_regret = float("inf") for epoch in range(1, args.epochs + 1): train_loss = train_epoch(