Rehome Weave Cookbooks

colabs/weave/Intro_to_Weave_Hello_Eval.ipynb

@@ -0,0 +1,164 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iYBudOac29pF"
+      },
+      "source": [
+        "# Introduction to Evaluations\n",
+        "\n",
+        "<img src=\"https://cdn.prod.website-files.com/62ba1fb86485b6d5029975c4/69b1b3d9c77f6e5294374be1_Endorsed_primary_goldblack.png\" width=\"400\" alt=\"Weights & Biases\" />\n",
+        "\n",
+        "Weave is a toolkit for developing AI-powered applications.\n",
+        "\n",
+        "This notebook demonstrates how to evaluate a model or function using Weave’s Evaluation API.\n",
+        "\n",
+        "In Weave, you evaluate your application by running it against a dataset of examples and scoring the outputs using custom-defined functions. This helps you to measure and improve your application's performance.\n",
+        "\n",
+        "In this notebook, you define a simple model, create a labeled dataset, track scoring functions with `@weave.op`, run an evaluation, and review the results in the Weave UI.\n",
+        "This workflow forms the foundation for more advanced workflows like fine tuning an LLM model, detecting regressions, and comparing models.\n",
+        "\n",
+        "To get started, complete the prerequisites. Then, define a Weave `Model` with a `predict` method, create a labeled dataset and scoring function, and run an evaluation using `weave.Evaluation.evaluate()`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2_KCsvwi3WHR"
+      },
+      "source": [
+        "## Run your first evaluation\n",
+        "\n",
+        "In this example, we're using Serverless Inference or OpenAI. [Learn more](/inference) about our inference API.\\\n",
+        "Using another provider? [We support all major clients and frameworks](https://docs.wandb.ai/weave/guides/integrations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-OjlLbir286L"
+      },
+      "outputs": [],
+      "source": [
+        "# Ensure your dependencies are installed with:\n",
+        "!pip install --quiet jedi openai pandas weave"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZgnPaanO3hMa"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import getpass\n",
+        "\n",
+        "#@title Set up your evaluation credentials\n",
+        "inference_provider = \"Serverless Inference\" #@param [\"Serverless Inference\", \"OpenAI\"]\n",
+        "\n",
+        "# Set up your W&B project and credentials\n",
+        "os.environ[\"WANDB_ENTITY_PROJECT\"] = input(\"Set up your W&B project (team name/project name): \")\n",
+        "os.environ[\"WANDB_API_KEY\"] = getpass.getpass(\"Set up your W&B API key (Create an API key at https://wandb.ai/settings): \")\n",
+        "\n",
+        "# Set up your OpenAI API key\n",
+        "if inference_provider == \"OpenAI\":\n",
+        "  os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"Enter your OpenAI API Key (Find it at https://platform.openai.com/api-keys): \")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EUdGUIVz4UYg"
+      },
+      "outputs": [],
+      "source": [
+        "import re\n",
+        "from textwrap import dedent\n",
+        "\n",
+        "from openai import OpenAI\n",
+        "\n",
+        "import weave\n",
+        "\n",
+        "class JsonModel(weave.Model):\n",
+        "    prompt: weave.Prompt = weave.StringPrompt(\n",
+        "        dedent(\"\"\"\n",
+        "You are an assistant that answers questions about JSON data provided by the user. The JSON data represents structured information of various kinds, and may be deeply nested. In the first user message, you will receive the JSON data under a label called 'context', and a question under a label called 'question'. Your job is to answer the question with as much accuracy and brevity as possible. Give only the answer with no preamble. You must output the answer in XML format, between <answer> and </answer> tags.\n",
+        "\"\"\")\n",
+        "    )\n",
+        "    if inference_provider == \"Serverless Inference\":\n",
+        "      model: str = \"OpenPipe/Qwen3-14B-Instruct\"\n",
+        "    if inference_provider == \"OpenAI\":\n",
+        "      model: str = \"gpt-4.1-nano\"\n",
+        "\n",
+        "    _client: OpenAI\n",
+        "\n",
+        "    def __init__(self):\n",
+        "        super().__init__()\n",
+        "        if inference_provider == \"Serverless Inference\":\n",
+        "          self._client = OpenAI(\n",
+        "              base_url=\"https://api.inference.wandb.ai/v1\",\n",
+        "              api_key=os.environ[\"WANDB_API_KEY\"],\n",
+        "              project=os.environ[\"WANDB_ENTITY_PROJECT\"],\n",
+        "          )\n",
+        "        if inference_provider == \"OpenAI\":\n",
+        "          self._client = OpenAI()\n",
+        "\n",
+        "    @weave.op\n",
+        "    def predict(self, context: str, question: str) -> str:\n",
+        "        response = self._client.chat.completions.create(\n",
+        "            model=self.model,\n",
+        "            messages=[\n",
+        "                {\"role\": \"system\", \"content\": self.prompt.format()},\n",
+        "                {\n",
+        "                    \"role\": \"user\",\n",
+        "                    \"content\": f\"Context: {context}\\nQuestion: {question}\",\n",
+        "                },\n",
+        "            ],\n",
+        "        )\n",
+        "        return response.choices[0].message.content\n",
+        "\n",
+        "@weave.op\n",
+        "def correct_answer_format(answer: str, output: str) -> dict[str, bool]:\n",
+        "    parsed_output = re.search(r\"<answer>(.*?)</answer>\", output, re.DOTALL)\n",
+        "    if parsed_output is None:\n",
+        "        return {\"correct_answer\": False, \"correct_format\": False}\n",
+        "    return {\"correct_answer\": parsed_output.group(1) == answer, \"correct_format\": True}\n",
+        "\n",
+        "if __name__ == \"__main__\":\n",
+        "    weave.init(os.environ[\"WANDB_ENTITY_PROJECT\"])\n",
+        "    model = JsonModel()\n",
+        "\n",
+        "    jsonqa = weave.Dataset.from_uri(\n",
+        "        \"weave:///wandb/json-qa/object/json-qa:v3\"\n",
+        "    ).to_pandas()\n",
+        "\n",
+        "    eval = weave.Evaluation(\n",
+        "        name=\"json-qa-eval\",\n",
+        "        dataset=weave.Dataset.from_pandas(jsonqa),\n",
+        "        scorers=[correct_answer_format],\n",
+        "    )\n",
+        "\n",
+        "    await eval.evaluate(model)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

colabs/weave/Intro_to_Weave_Hello_Trace.ipynb

@@ -0,0 +1,111 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8ZmKBnUE8bGR"
+      },
+      "source": [
+        "# Introduction to Traces\n",
+        "\n",
+        "<img src=\"https://cdn.prod.website-files.com/62ba1fb86485b6d5029975c4/69b1b3d9c77f6e5294374be1_Endorsed_primary_goldblack.png\" width=\"400\" alt=\"Weights & Biases\" />\n",
+        "\n",
+        "Weave is a toolkit for developing AI-powered applications.\n",
+        "\n",
+        "Use Weave traces to capture the inputs, outputs, and internal structure of your Python function automatically to observe and debug LLM applications.\n",
+        "\n",
+        "When you decorate a function with `@weave.op`, Weave records a rich trace of the function while it runs, including any nested operations or external API calls. Use the trace to to debug, understand, and visualize interactions between your code and LLM models, without leaving your notebook.\n",
+        "\n",
+        "To get started, complete the prerequisites. Then, define a function decorated with `@weave.op` decorator and run it on an example input to track LLM calls. Weave captures and visualizes the trace automatically."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jxECTbwy8cvT"
+      },
+      "outputs": [],
+      "source": [
+        "# Ensure your dependencies are installed with:\n",
+        "!pip install --quiet jedi openai weave"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ujOegcPY8j7z"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import getpass\n",
+        "\n",
+        "#@title Set up your credentials\n",
+        "inference_provider = \"Serverless Inference\" #@param [\"Serverless Inference\", \"OpenAI\"]\n",
+        "\n",
+        "# Set up your W&B project and credentials\n",
+        "os.environ[\"WANDB_ENTITY_PROJECT\"] = input(\"Set up your W&B project (team name/project name): \")\n",
+        "os.environ[\"WANDB_API_KEY\"] = getpass.getpass(\"Set up your W&B API key (Create an API key at https://wandb.ai/settings): \")\n",
+        "\n",
+        "# Set up your OpenAI API key\n",
+        "if inference_provider == \"OpenAI\":\n",
+        "  os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"Enter your OpenAI API Key (Find it at https://platform.openai.com/api-keys): \")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pTUHuulv8afx"
+      },
+      "outputs": [],
+      "source": [
+        "from openai import OpenAI\n",
+        "import weave\n",
+        "\n",
+        "weave.init(os.environ[\"WANDB_ENTITY_PROJECT\"])\n",
+        "\n",
+        "@weave.op  # Decorator to track requests\n",
+        "def create_completion(message: str) -> str:\n",
+        "    if inference_provider == \"Serverless Inference\":\n",
+        "      client = OpenAI(\n",
+        "          base_url=\"https://api.inference.wandb.ai/v1\",\n",
+        "          api_key=os.environ[\"WANDB_API_KEY\"],\n",
+        "          project=os.environ[\"WANDB_ENTITY_PROJECT\"],\n",
+        "      )\n",
+        "      model_name: str = \"OpenPipe/Qwen3-14B-Instruct\"\n",
+        "    if inference_provider == \"OpenAI\":\n",
+        "      client = OpenAI()\n",
+        "      model_name: str = \"gpt-4.1-nano\"\n",
+        "    response = client.chat.completions.create(\n",
+        "        model=model_name,\n",
+        "        messages=[\n",
+        "            {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
+        "            {\"role\": \"user\", \"content\": message},\n",
+        "        ],\n",
+        "    )\n",
+        "    return response.choices[0].message.content\n",
+        "\n",
+        "\n",
+        "message = \"Tell me a joke.\"\n",
+        "create_completion(message)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}