diff --git a/.gitignore b/.gitignore deleted file mode 100644 index fe1f6e5..0000000 --- a/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -venv -.idea -example.py -parents.csv -children.csv -images.csv \ No newline at end of file diff --git a/Fetch.py b/Fetch.py new file mode 100644 index 0000000..1a8c3df --- /dev/null +++ b/Fetch.py @@ -0,0 +1,64 @@ +import requests +import pandas as pd + +# ── CONFIG ──────────────────────────────────────────────────────────────────── +SHOP_URL = "https://www.peepultree.world" # Replace with target store URL + +def fetch_all_products(shop_url): + products_list = [] + page = 1 + + while True: + url = f"{shop_url}/products.json?limit=250&page={page}" + response = requests.get(url) + + if response.status_code != 200: + print(f"Failed to fetch page {page}: {response.status_code}") + break + + data = response.json().get("products", []) + + if not data: + break + + for product in data: + title = product["title"] + + # ── Collect ALL image URLs for this product ──────────────────── + images = product.get("images", []) + all_photo_urls = " | ".join([img["src"] for img in images]) + # URLs are separated by " | " so the CSV stays readable + # Example: "https://cdn.../img1.jpg | https://cdn.../img2.jpg" + + if not all_photo_urls: + all_photo_urls = "No images" + + # ── Each variant gets its own row ────────────────────────────── + for variant in product["variants"]: + products_list.append({ + "Product Name" : title, + "Variant" : variant["title"], + "Price" : float(variant["price"]), + "Inventory" : variant.get("inventory_quantity", "N/A"), + "Product ID" : variant["id"], + "All Photos" : all_photo_urls # All URLs in one cell + }) + + print(f"Fetched page {page} — {len(data)} products") + page += 1 + + return products_list + + +def save_to_csv(products_list): + df = pd.DataFrame(products_list) + df.index += 1 + df.to_csv("products.csv", index_label="Row") + print(f"\n✅ Saved {len(products_list)} products to products.csv") + print("📸 All photo URLs saved in the 'All Photos' column, separated by ' | '") + + +if __name__ == "__main__": + print("Fetching products...") + products = fetch_all_products(SHOP_URL) + save_to_csv(products) diff --git a/README.md b/README.md index 242125b..221b081 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,206 @@ -# ShopifyScraper +# 🛒 Shopify Product Tracker -ShopifyScraper is a Python package that scrapes data from Shopify. Unlike a regular web scraper, that needs to visit every page on a site, this package fetches Shopify's publicly visible `products.json` file, allowing you to scrape an entire store inventory in seconds. +A Python toolkit to fetch, monitor, and download product data from any public Shopify store — no API key required. -When the commands below are run, ShopifyScraper will extract the store inventory and save the products and product variants to Pandas dataframes, from where you can access or analyse the data or write it to CSV or database. +Track price changes, monitor inventory, and bulk-download product images all from the command line. -### Installation -To install ShopifyScraper, run the following command: +--- + +## ✨ Features + +- 📋 Fetch complete product listings (name, variant, price, inventory, images) +- 💾 Save everything to a clean, organised CSV +- 📈 Track price changes on selected products across runs +- 🟢 Visual indicators for price drops, rises, and no change +- 📸 Bulk download all product images into organised folders +- ⏭️ Smart skip — won't re-download images that already exist +- 🌐 Works on any public Shopify store without an API key + +--- + +## 📁 Project Structure + +``` +ShopifyTracker/ +│ +├── fetch.py # Fetches all products and saves to CSV +├── tracker.py # Checks current prices against saved prices +├── downloader.py # Downloads product images from the CSV +│ +├── products.csv # Auto-generated — full product list +├── tracked.csv # Auto-generated — your chosen tracked items +│ +└── product_images/ # Auto-generated — downloaded images + ├── Product Name/ + │ └── Variant/ + │ ├── image1.jpg + │ └── image2.jpg + └── ... +``` + +--- + +## ⚙️ Installation + +**Requirements:** Python 3.8+ ```bash -pip3 install git+https://github.com/practical-data-science/ShopifyScraper.git +pip install requests pandas ``` -### Usage +--- + +## 🚀 How to Use + +### Step 1 — Fetch all products + +Set your store URL in `fetch.py`: ```python -from shopify_scraper import scraper +SHOP_URL = "https://yourstore.myshopify.com" +``` + +Then run: + +```bash +python fetch.py +``` -url = "https://yourshopifydomain.com" +This generates `products.csv` with every product listed on the store. -parents = scraper.get_products(url) -parents.to_csv('parents.csv', index=False) -print('Parents: ', len(parents)) +--- +### Step 2 — Track prices -children = scraper.get_variants(parents) -children.to_csv('children.csv', index=False) -print('Children: ', len(children)) +Open `products.csv` and note the **Row numbers** of items you want to monitor. +Set the same store URL in `tracker.py`, then run: -images = scraper.get_images(parents) -images.to_csv('images.csv', index=False) -print('Images: ', len(images)) +```bash +python tracker.py +``` +Enter your chosen row numbers when prompted: + +``` +Enter row numbers to track (from products.csv) +Example: 1 4 7 12 +Row numbers: 2 5 9 ``` +--- + +### Step 3 — Download images *(optional)* + +```bash +python downloader.py +``` + +Enter row numbers or type `all` to download every product's images: + +``` +Row numbers: all +``` + +Images are saved to `product_images/` organised by product and variant. + +--- + +## 📊 Sample Output + +### fetch.py +``` +Fetching products... +Fetched page 1 — 250 products +Fetched page 2 — 143 products + +✅ Saved 393 products to products.csv +📸 All photo URLs saved in the 'All Photos' column, separated by ' | ' +``` + +### tracker.py +``` +Tracking 3 item(s) — 2024-11-01 10:30:00 + +Row Product Saved Price Current Price Change +------------------------------------------------------------------------------ +2 Nike Air Max 90 ₹4999 ₹4499 🟢 DROP ₹500.00 +5 Adidas Ultraboost 22 ₹6999 ₹6999 ⚪ No change +9 Puma RS-X ₹3499 ₹3799 🔴 RISE ₹300.00 +``` + +### downloader.py +``` +📦 Row 2 — Nike Air Max 90 (Size 9): 3 image(s) + ✅ [1/3] Downloaded — image1.jpg + ✅ [2/3] Downloaded — image2.jpg + ✅ [3/3] Downloaded — image3.jpg + +────────────────────────────────────────── +✅ Downloaded : 3 image(s) +❌ Failed : 0 image(s) +📁 Saved to : /ShopifyTracker/product_images +``` + +--- + +## 🗂️ CSV Format + +| Row | Product Name | Variant | Price | Inventory | Product ID | All Photos | +|-----|-------------|---------|-------|-----------|------------|------------| +| 1 | Nike Air Max | Size 9 | 4999 | 50 | 123456 | https://cdn/.../1.jpg \| https://cdn/.../2.jpg | +| 2 | Adidas Ultra | Size 10 | 6999 | 30 | 789012 | https://cdn/.../3.jpg | + +> **All Photos** column stores multiple image URLs separated by ` | ` +> Copy any URL and paste into your browser to view the image directly. + +--- + +## 🧠 Tech Stack + +| Tool | Purpose | +|------|---------| +| [Requests](https://docs.python-requests.org/) | Fetching data from Shopify's JSON API | +| [pandas](https://pandas.pydata.org/) | Reading and writing CSV files | +| [os / re](https://docs.python.org/3/library/os.html) | Folder creation and filename sanitisation | +| Shopify `/products.json` | Public product data endpoint | + +--- + +## ⚙️ Configuration + +| Variable | File | Default | Description | +|----------|------|---------|-------------| +| `SHOP_URL` | `fetch.py`, `tracker.py` | — | Target Shopify store URL | +| `CSV_FILE` | `downloader.py` | `products.csv` | Path to your product CSV | +| `OUTPUT_DIR` | `downloader.py` | `product_images` | Folder to save downloaded images | + +--- + +## ⚠️ Known Limitations + +- Only works on Shopify stores that have **public product listings enabled**. Stores with password protection or restricted APIs will return a `403` or `404` error. +- Inventory quantity may show `N/A` if the store has hidden stock levels. +- Shopify's `/products.json` endpoint returns a maximum of **250 products per page** — the script handles pagination automatically. + +--- + +## 💡 Possible Improvements + +- 📧 Email alerts when a tracked price drops +- 📊 Price history graph using matplotlib +- 🗃️ SQLite database instead of CSV for larger stores +- 🌐 Flask web dashboard to manage tracked products +- ⏰ Scheduled auto-runs using the `schedule` library + +--- + +## 📄 License + +This project is open source and available under the [MIT License](LICENSE). + +--- + +## 🤝 Contributing + +Pull requests are welcome! If you find a store where the scraper breaks or have an improvement in mind, feel free to open an issue. diff --git a/Tracker.py b/Tracker.py new file mode 100644 index 0000000..96d1d14 --- /dev/null +++ b/Tracker.py @@ -0,0 +1,66 @@ +import requests +import pandas as pd +from datetime import datetime + +SHOP_URL = "https://yourstore.myshopify.com" # Same store URL as fetch.py + +def get_current_price(product_id): + """ + Fetches the latest price of a variant directly from Shopify's API + using the product variant ID saved in the CSV. + """ + url = f"{SHOP_URL}/products.json?limit=250" + response = requests.get(url) + + if response.status_code != 200: + return None + + for product in response.json().get("products", []): + for variant in product["variants"]: + if variant["id"] == product_id: + return float(variant["price"]) + return None + + +def track_prices(row_numbers): + """ + Loads products.csv, filters chosen rows, checks current prices, + and reports any changes. + """ + df = pd.read_csv("products.csv", index_col="Row") + + chosen = df.loc[row_numbers] # Filter only chosen rows + + # Save chosen items to tracked.csv on first run + chosen.to_csv("tracked.csv") + print(f"\nTracking {len(chosen)} item(s) — {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") + print(f"{'Row':<5} {'Product':<40} {'Saved Price':<15} {'Current Price':<15} {'Change'}") + print("-" * 90) + + for row, item in chosen.iterrows(): + saved_price = item["Price"] + product_id = item["Product ID"] + current_price = get_current_price(int(product_id)) + + if current_price is None: + print(f"{row:<5} {item['Product Name']:<40} ₹{saved_price:<14} Could not fetch") + continue + + # Calculate change + change = current_price - saved_price + if change < 0: + status = f"🟢 DROP ₹{abs(change):.2f}" + elif change > 0: + status = f"🔴 RISE ₹{change:.2f}" + else: + status = "⚪ No change" + + print(f"{row:<5} {item['Product Name']:<40} ₹{saved_price:<14} ₹{current_price:<14} {status}") + + +if __name__ == "__main__": + print("Enter row numbers to track (from products.csv)") + print("Example: 1 4 7 12") + user_input = input("Row numbers: ").strip().split() + row_numbers = [int(r) for r in user_input] + track_prices(row_numbers) diff --git a/downloader.py b/downloader.py new file mode 100644 index 0000000..7dc903f --- /dev/null +++ b/downloader.py @@ -0,0 +1,108 @@ +import requests +import pandas as pd +import os +import re + +# ── CONFIG ──────────────────────────────────────────────────────────────────── +CSV_FILE = "products.csv" # Your product CSV +OUTPUT_DIR = "product_images" # Folder where images will be saved + +def sanitise_name(name): + """ + Removes special characters from product names so they + can be safely used as folder names on any OS. + """ + return re.sub(r'[\\/*?:"<>|]', "", name).strip() + + +def download_images_for_rows(row_numbers): + """ + Reads the CSV, filters chosen rows, and downloads + all photos for each product into its own subfolder. + """ + df = pd.read_csv(CSV_FILE, index_col="Row") + + # Validate row numbers + valid_rows = [r for r in row_numbers if r in df.index] + invalid_rows = [r for r in row_numbers if r not in df.index] + + if invalid_rows: + print(f"⚠️ Skipping invalid rows: {invalid_rows}") + + if not valid_rows: + print("❌ No valid rows found. Exiting.") + return + + chosen = df.loc[valid_rows] + + # Create main output directory + os.makedirs(OUTPUT_DIR, exist_ok=True) + + total_downloaded = 0 + total_failed = 0 + + for row, item in chosen.iterrows(): + product_name = sanitise_name(item["Product Name"]) + variant = sanitise_name(str(item["Variant"])) + photo_urls = item["All Photos"] + + if pd.isna(photo_urls) or photo_urls == "No images": + print(f"\n⚠️ Row {row} — {product_name} ({variant}): No images found, skipping.") + continue + + # Split the " | " separated URLs back into a list + urls = [url.strip() for url in photo_urls.split("|")] + + # Create a subfolder per product variant + # e.g. product_images/Nike Air Max/Size 9/ + folder = os.path.join(OUTPUT_DIR, product_name, variant) + os.makedirs(folder, exist_ok=True) + + print(f"\n📦 Row {row} — {product_name} ({variant}): {len(urls)} image(s)") + + for i, url in enumerate(urls, start=1): + # Derive a clean filename from the URL + # e.g. https://cdn.shopify.com/.../image.jpg → image.jpg + filename = url.split("?")[0].split("/")[-1] # Strip query params + save_path = os.path.join(folder, filename) + + # Skip if already downloaded + if os.path.exists(save_path): + print(f" ⏭️ [{i}/{len(urls)}] Already exists — {filename}") + continue + + try: + response = requests.get(url, timeout=10) + if response.status_code == 200: + with open(save_path, "wb") as f: + f.write(response.content) + print(f" ✅ [{i}/{len(urls)}] Downloaded — {filename}") + total_downloaded += 1 + else: + print(f" ❌ [{i}/{len(urls)}] Failed (HTTP {response.status_code}) — {url}") + total_failed += 1 + except requests.exceptions.RequestException as e: + print(f" ❌ [{i}/{len(urls)}] Error — {e}") + total_failed += 1 + + # ── Summary ─────────────────────────────────────────────────────────────── + print("\n" + "─" * 50) + print(f"✅ Downloaded : {total_downloaded} image(s)") + print(f"❌ Failed : {total_failed} image(s)") + print(f"📁 Saved to : {os.path.abspath(OUTPUT_DIR)}") + + +if __name__ == "__main__": + print("Enter row numbers to download images for (from products.csv)") + print("Example: 1 4 7 12") + print("Type 'all' to download images for every product\n") + + user_input = input("Row numbers: ").strip() + + if user_input.lower() == "all": + df = pd.read_csv(CSV_FILE, index_col="Row") + row_numbers = list(df.index) + else: + row_numbers = [int(r) for r in user_input.split()] + + download_images_for_rows(row_numbers) diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index a94cf69..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests -pandas \ No newline at end of file diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b88034e..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -description-file = README.md diff --git a/setup.py b/setup.py deleted file mode 100644 index b3ac180..0000000 --- a/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -from setuptools import setup - -from os import path -this_directory = path.abspath(path.dirname(__file__)) -with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: - long_description = f.read() - -setup( - name='ShopifyScraper', - packages=['shopify_scraper'], - version='0.002', - license='MIT', - description='ShopifyScraper is a Python package that scrapes Shopify products and variants to Pandas dataframes.', - long_description=long_description, - long_description_content_type='text/markdown', - author='Matt Clarke', - author_email='matt@practicaldatascience.co.uk', - url='https://github.com/practical-data-science/ShopifyScraper', - download_url='https://github.com/practical-data-science/ShopifyScraper/archive/master.zip', - keywords=['python', 'requests', 'pandas', 'shopify', 'scraping'], - classifiers=[ - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3.8', - ], - install_requires=['pandas', 'requests'] -) \ No newline at end of file diff --git a/shopify_scraper/scraper.py b/shopify_scraper/scraper.py index a2973ed..c3408ca 100644 --- a/shopify_scraper/scraper.py +++ b/shopify_scraper/scraper.py @@ -121,7 +121,7 @@ def json_list_to_df(df, col): """ rows = [] - for index, row in df[col].iteritems(): + for index, row in df[col].items(): for item in row: rows.append(item) df = pd.DataFrame(rows)