Lesson 1: import Olist data and plant evaluation scenarios

.github/workflows/ci.yml

@@ -96,57 +96,12 @@ jobs:
         with:
           bundler-cache: true
 
-      - name: Run tests
+      - name: Run specs
         env:
           RAILS_ENV: test
           DATABASE_URL: postgres://postgres:postgres@localhost:5432
           # RAILS_MASTER_KEY: ${{ secrets.RAILS_MASTER_KEY }}
           # REDIS_URL: redis://localhost:6379/0
-        run: bin/rails db:test:prepare test
-
-  system-test:
-    runs-on: ubuntu-latest
-
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-        ports:
-          - 5432:5432
-        options: --health-cmd="pg_isready" --health-interval=10s --health-timeout=5s --health-retries=3
-
-      # redis:
-      #   image: valkey/valkey:8
-      #   ports:
-      #     - 6379:6379
-      #   options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
-
-    steps:
-      - name: Install packages
-        run: sudo apt-get update && sudo apt-get install --no-install-recommends -y libpq-dev libvips
-
-      - name: Checkout code
-        uses: actions/checkout@v6
-
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true
-
-      - name: Run System Tests
-        env:
-          RAILS_ENV: test
-          DATABASE_URL: postgres://postgres:postgres@localhost:5432
-          # RAILS_MASTER_KEY: ${{ secrets.RAILS_MASTER_KEY }}
-          # REDIS_URL: redis://localhost:6379/0
-        run: bin/rails db:test:prepare test:system
-
-      - name: Keep screenshots from failed system tests
-        uses: actions/upload-artifact@v4
-        if: failure()
-        with:
-          name: screenshots
-          path: ${{ github.workspace }}/tmp/screenshots
-          if-no-files-found: ignore
+        run: |
+          bin/rails db:test:prepare
+          bundle exec rspec

.rspec

@@ -0,0 +1 @@
+--require spec_helper

Gemfile

@@ -54,6 +54,9 @@ group :development, :test do
 
   # Omakase Ruby styling [https://github.com/rails/rubocop-rails-omakase/]
   gem "rubocop-rails-omakase", require: false
+  gem "rspec-rails"
+  gem "factory_bot_rails"
+  gem "faker"
 end
 
 group :development do
@@ -66,3 +69,5 @@ group :test do
   gem "capybara"
   gem "selenium-webdriver"
 end
+
+gem "csv"

Gemfile.lock

@@ -102,17 +102,26 @@ GEM
     concurrent-ruby (1.3.8)
     connection_pool (3.0.2)
     crass (1.0.7)
+    csv (3.3.6)
     date (3.5.1)
     debug (1.11.1)
       irb (~> 1.10)
       reline (>= 0.3.8)
+    diff-lcs (1.6.2)
     dotenv (3.2.0)
     drb (2.2.3)
     ed25519 (1.4.0)
     erb (6.0.7)
     erubi (1.13.1)
     et-orbi (1.4.2)
       tzinfo
+    factory_bot (6.6.0)
+      activesupport (>= 6.1.0)
+    factory_bot_rails (6.5.1)
+      factory_bot (~> 6.5)
+      railties (>= 6.1.0)
+    faker (3.8.0)
+      i18n (>= 1.8.11, < 2)
     ffi (1.17.4-aarch64-linux-gnu)
     ffi (1.17.4-aarch64-linux-musl)
     ffi (1.17.4-arm-linux-gnu)
@@ -277,6 +286,23 @@ GEM
     regexp_parser (2.13.1)
     reline (0.7.0)
       io-console (~> 0.5)
+    rspec-core (3.13.6)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.8)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-rails (8.0.4)
+      actionpack (>= 7.2)
+      activesupport (>= 7.2)
+      railties (>= 7.2)
+      rspec-core (>= 3.13.0, < 5.0.0)
+      rspec-expectations (>= 3.13.0, < 5.0.0)
+      rspec-mocks (>= 3.13.0, < 5.0.0)
+      rspec-support (>= 3.13.0, < 5.0.0)
+    rspec-support (3.13.7)
     rubocop (1.91.0)
       json (>= 2.3)
       language_server-protocol (~> 3.17.0.2)
@@ -393,7 +419,10 @@ DEPENDENCIES
   brakeman
   bundler-audit
   capybara
+  csv
   debug
+  factory_bot_rails
+  faker
   image_processing (~> 1.2)
   importmap-rails
   jbuilder
@@ -402,6 +431,7 @@ DEPENDENCIES
   propshaft
   puma (>= 5.0)
   rails (~> 8.1.4)
+  rspec-rails
   rubocop-rails-omakase
   selenium-webdriver
   solid_cable
@@ -441,14 +471,19 @@ CHECKSUMS
   concurrent-ruby (1.3.8) sha256=b2f1be836e968ccc78ccfce277ea79c72a88633f22306782c16ff23fb415d1e1
   connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a
   crass (1.0.7) sha256=94868719948664c89ddcaf0a37c65048413dfcb1c869470a5f7a7ceb5390b295
+  csv (3.3.6) sha256=aba61e7e507a66f03d45cb1f3c4b6359861c3504038b422962875dce099e4456
   date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0
   debug (1.11.1) sha256=2e0b0ac6119f2207a6f8ac7d4a73ca8eb4e440f64da0a3136c30343146e952b6
+  diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962
   dotenv (3.2.0) sha256=e375b83121ea7ca4ce20f214740076129ab8514cd81378161f11c03853fe619d
   drb (2.2.3) sha256=0b00d6fdb50995fe4a45dea13663493c841112e4068656854646f418fda13373
   ed25519 (1.4.0) sha256=16e97f5198689a154247169f3453ef4cfd3f7a47481fde0ae33206cdfdcac506
   erb (6.0.7) sha256=c5ca6dc25b0ef974a44dc8f59fe847577122483b1968a38dec305c60bf91ee92
   erubi (1.13.1) sha256=a082103b0885dbc5ecf1172fede897f9ebdb745a4b97a5e8dc63953db1ee4ad9
   et-orbi (1.4.2) sha256=bb555dae668419cb24caa2a293a170e58be6d4df1e017c51f5030bdc133cd20c
+  factory_bot (6.6.0) sha256=1fc1b3b5620ec980a6a27aec1b6ec8c250ca82962e970e8a40f93e8d388d4b89
+  factory_bot_rails (6.5.1) sha256=d3cc4851eae4dea8a665ec4a4516895045e710554d2b5ac9e68b94d351bc6d68
+  faker (3.8.0) sha256=c147b308df73a90f27a4fc84f18d4c22ef0ad9c2a64b2b61c86fd0ca71753efc
   ffi (1.17.4-aarch64-linux-gnu) sha256=b208f06f91ffd8f5e1193da3cae3d2ccfc27fc36fba577baf698d26d91c080df
   ffi (1.17.4-aarch64-linux-musl) sha256=9286b7a615f2676245283aef0a0a3b475ae3aae2bb5448baace630bb77b91f39
   ffi (1.17.4-arm-linux-gnu) sha256=d6dbddf7cb77bf955411af5f187a65b8cd378cb003c15c05697f5feee1cb1564
@@ -520,6 +555,11 @@ CHECKSUMS
   rdoc (8.1.0) sha256=befc37880a5cfd36dcb176c748b5fd5ee39dd3536a75e81824e4972ce230f60a
   regexp_parser (2.13.1) sha256=5aedb6b7c35688f51e86eef17a15374dfd287c83f4ca644b2c5f3ec50a33b44b
   reline (0.7.0) sha256=5b012d8e55dbf9d450f12bde2cf7d15ff546ae80b3f8f3b30e570d431815583d
+  rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d
+  rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836
+  rspec-mocks (3.13.8) sha256=086ad3d3d17533f4237643de0b5c42f04b66348c28bf6b9c2d3f4a3b01af1d47
+  rspec-rails (8.0.4) sha256=06235692fc0892683d3d34977e081db867434b3a24ae0dd0c6f3516bad4e22df
+  rspec-support (3.13.7) sha256=0640e5570872aafefd79867901deeeeb40b0c9875a36b983d85f54fb7381c47c
   rubocop (1.91.0) sha256=9c82b7bf391c5d7e3798c5b9996e22a1fe3bd7468e351dfdeb96140c058296d0
   rubocop-ast (1.50.0) sha256=b9ca88300da0803ee222ad20cdb30494c0a784eed06fdc35d254b06d662788db
   rubocop-performance (1.27.0) sha256=eeeb1374d062a368ee1c787b70eb0b0cc4b184cb1f8565f424760946146d61ce

README.md

@@ -104,12 +104,23 @@ bin/rails db:migrate
 bin/dev
 ```
 
-The dataset import task and the scenario seed script are part of Lesson 1 and do not exist yet.
+Put the Olist files in `data/olist/` (see the Data section), then load them:
+
+```bash
+LIMIT=200 bin/rails data:import_olist   # small sample: 200 customers and their orders, about half a minute
+bin/rails data:import_olist             # the full dataset, about 3 minutes
+bin/rails data:plant_scenarios          # shipments for every order, plus the planted scenarios
+```
+
+Each import step prints how many rows it loaded and how many it skipped. `data:plant_scenarios` first generates the shipments, then edits five fixed orders (the first five that have a shipment) so each has a known answer for the evals. It prints the order id of each scenario, and running it again gives the same result.
 
 ## Scope
 
 This is a learning project about building a reliable AI agent, not a complete support product. Real Shopify or BigCommerce integration, multi-modal input and new eval tooling are out of scope for now.
 
 ## More
 
-The full plan, with architecture, gems by lesson, data mapping and the evaluation plan, is in [docs/PROJECT_PLAN.md](docs/PROJECT_PLAN.md).
+- The full plan, with architecture, gems by lesson, data mapping and the evaluation plan: [docs/PROJECT_PLAN.md](docs/PROJECT_PLAN.md)
+- The database tables and how they relate, as a diagram: [docs/DATA_MODEL.md](docs/DATA_MODEL.md)
+- Lesson progress, decisions and what broke: [docs/lessons/01-data-model-and-import.md](docs/lessons/01-data-model-and-import.md)
+- Index of all documentation: [docs/README.md](docs/README.md)

app/models/customer.rb

@@ -0,0 +1,3 @@
+class Customer < ApplicationRecord
+  has_many :orders
+end

app/models/order.rb

@@ -0,0 +1,7 @@
+class Order < ApplicationRecord
+  belongs_to :customer
+  has_many :order_items, dependent: :destroy
+  has_many :products, through: :order_items
+  has_many :payments, dependent: :destroy
+  has_one :shipment, dependent: :destroy
+end

app/models/order_item.rb

@@ -0,0 +1,4 @@
+class OrderItem < ApplicationRecord
+  belongs_to :order
+  belongs_to :product
+end

app/models/payment.rb

@@ -0,0 +1,3 @@
+class Payment < ApplicationRecord
+  belongs_to :order
+end

app/models/product.rb

@@ -0,0 +1,4 @@
+class Product < ApplicationRecord
+  has_many :order_items
+  has_many :orders, through: :order_items
+end

app/models/shipment.rb

@@ -0,0 +1,3 @@
+class Shipment < ApplicationRecord
+  belongs_to :order
+end

db/migrate/20261006134215_create_customers.rb

@@ -0,0 +1,14 @@
+class CreateCustomers < ActiveRecord::Migration[8.1]
+  def change
+    create_table :customers do |t|
+      t.string :olist_customer_unique_id, null: false
+      t.string :name
+      t.string :email
+      t.string :city
+      t.string :state
+
+      t.timestamps
+    end
+    add_index :customers, :olist_customer_unique_id, unique: true
+  end
+end

db/migrate/20261006135202_create_products.rb

@@ -0,0 +1,19 @@
+class CreateProducts < ActiveRecord::Migration[8.1]
+  def change
+    create_table :products do |t|
+      t.string :olist_product_id, null: false
+      t.string :name
+      t.string :category
+      t.integer :name_length
+      t.integer :description_length
+      t.integer :photos_count
+      t.integer :weight_g
+      t.integer :length_cm
+      t.integer :height_cm
+      t.integer :width_cm
+
+      t.timestamps
+    end
+    add_index :products, :olist_product_id, unique: true
+  end
+end

db/migrate/20261006141029_create_orders.rb

@@ -0,0 +1,17 @@
+class CreateOrders < ActiveRecord::Migration[8.1]
+  def change
+    create_table :orders do |t|
+      t.string :olist_order_id, null: false
+      t.references :customer, null: false, foreign_key: true
+      t.string :status, null: false
+      t.datetime :purchased_at
+      t.datetime :approved_at
+      t.datetime :shipped_at
+      t.datetime :delivered_at
+      t.datetime :estimated_delivery_at
+
+      t.timestamps
+    end
+    add_index :orders, :olist_order_id, unique: true
+  end
+end

db/migrate/20261006141535_create_order_items.rb

@@ -0,0 +1,12 @@
+class CreateOrderItems < ActiveRecord::Migration[8.1]
+  def change
+    create_table :order_items do |t|
+      t.references :order, null: false, foreign_key: true
+      t.references :product, null: false, foreign_key: true
+      t.decimal :price, precision: 10, scale: 2
+      t.decimal :freight_value, precision: 10, scale: 2
+
+      t.timestamps
+    end
+  end
+end

db/migrate/20261006142602_create_payments.rb

@@ -0,0 +1,13 @@
+class CreatePayments < ActiveRecord::Migration[8.1]
+  def change
+    create_table :payments do |t|
+      t.references :order, null: false, foreign_key: true
+      t.integer :sequence
+      t.string :payment_method
+      t.integer :installments
+      t.decimal :amount, null: false, precision: 10, scale: 2
+
+      t.timestamps
+    end
+  end
+end

db/migrate/20261006143358_create_shipments.rb

@@ -0,0 +1,14 @@
+class CreateShipments < ActiveRecord::Migration[8.1]
+  def change
+    create_table :shipments do |t|
+      t.references :order, null: false, foreign_key: true, index: { unique: true }
+      t.string :carrier
+      t.string :tracking_number
+      t.string :status, null: false
+      t.string :last_known_location
+      t.datetime :last_scan_at
+
+      t.timestamps
+    end
+  end
+end

db/migrate/20261006144548_add_unique_index_to_customers_email.rb

@@ -0,0 +1,5 @@
+class AddUniqueIndexToCustomersEmail < ActiveRecord::Migration[8.1]
+  def change
+    add_index :customers, :email, unique: true
+  end
+end

db/migrate/20261007133706_add_final_sale_to_products.rb

@@ -0,0 +1,5 @@
+class AddFinalSaleToProducts < ActiveRecord::Migration[8.1]
+  def change
+    add_column :products, :final_sale, :boolean, null: false, default: false
+  end
+end