From 69e99dcc393c21bbd5e0a34774e97d9e31c4b4a4 Mon Sep 17 00:00:00 2001 From: jdatjedd <130353431+jdatjedd@users.noreply.github.com> Date: Fri, 30 Aug 2024 10:32:51 +0530 Subject: [PATCH 01/12] Update scraper.py --- shopify_scraper/scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shopify_scraper/scraper.py b/shopify_scraper/scraper.py index a2973ed..c3408ca 100644 --- a/shopify_scraper/scraper.py +++ b/shopify_scraper/scraper.py @@ -121,7 +121,7 @@ def json_list_to_df(df, col): """ rows = [] - for index, row in df[col].iteritems(): + for index, row in df[col].items(): for item in row: rows.append(item) df = pd.DataFrame(rows) From f2490d0095a9a884fba3f042219fac6e96412bd2 Mon Sep 17 00:00:00 2001 From: jdatjedd <130353431+jdatjedd@users.noreply.github.com> Date: Fri, 30 Aug 2024 10:59:17 +0530 Subject: [PATCH 02/12] Create all in one file You just need to copy this file into your code editor and chage the url in the bottom --- all in one file | 132 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 all in one file diff --git a/all in one file b/all in one file new file mode 100644 index 0000000..597fbbd --- /dev/null +++ b/all in one file @@ -0,0 +1,132 @@ +""" +Shopify scraper +Description: Scrapes products from a Shopify store by parsing products.json and converting it to a pandas DataFrame. +Author: Matt Clarke +""" + +import json +import pandas as pd +import requests + +def get_json(url: str, page: int) -> str: + """ + Get Shopify products.json from a store URL. + + Args: + url (str): URL of the store. + page (int): Page number of the products.json. + Returns: + products_json: Products.json from the store. + """ + + response = requests.get(f'{url}/products.json?limit=250&page={page}', timeout=5) + products_json = response.text + response.raise_for_status() + return products_json + +def _products_json_to_df(products_json: str) -> pd.DataFrame: + """ + Convert products.json to a pandas DataFrame. + + Args: + products_json (json): Products.json from the store. + Returns: + df: Pandas DataFrame of the products.json. + """ + + products_dict = json.loads(products_json) + df = pd.DataFrame.from_dict(products_dict['products']) + return df + +def get_products(url: str) -> pd.DataFrame: + """ + Get all products from a store. + + Returns: + df: Pandas DataFrame of the products.json. + """ + + results = True + page = 1 + df = pd.DataFrame() + + while results: + products_json = get_json(url, page) + products_dict = _products_json_to_df(products_json) + + if len(products_dict) == 0: + break + else: + df = pd.concat([df, products_dict], ignore_index=True) + page += 1 + + df['url'] = f"{url}/products/" + df['handle'] + return df + +def get_variants(products: pd.DataFrame) -> pd.DataFrame: + """Get variants from a table of products. + + Args: + products (pd.DataFrame): Pandas dataframe of products from get_products() + + Returns: + variants (pd.DataFrame): Pandas dataframe of variants + """ + + products['id'].astype(int) + df_variants = pd.DataFrame() + + for row in products.itertuples(index='True'): + for variant in getattr(row, 'variants'): + df_variants = pd.concat([df_variants, pd.DataFrame.from_records(variant, index=[0])]) + + df_variants['id'].astype(int) + df_variants['product_id'].astype(int) + df_product_data = products[['id', 'title', 'vendor']] + df_product_data = df_product_data.rename(columns={'title': 'product_title', 'id': 'product_id', 'vendor': 'product_vendor'}) + df_variants = df_variants.merge(df_product_data, left_on='product_id', right_on='product_id', validate='m:1') + return df_variants + +def _flatten_column_to_dataframe(df: pd.DataFrame, col: str) -> pd.DataFrame: + """Return a Pandas dataframe based on a column that contains a list of JSON objects. + + Args: + df (Pandas dataframe): The dataframe to be flattened. + col (str): The name of the column that contains the JSON objects. + + Returns: + Pandas dataframe: A new dataframe with the JSON objects expanded into columns. + """ + + rows = [item for row in df[col] for item in row] + return pd.DataFrame(rows) + +def get_images(df_products: pd.DataFrame) -> pd.DataFrame: + """Get images from a list of products. + + Args: + df_products (pd.DataFrame): Pandas dataframe of products from get_products() + + Returns: + images (pd.DataFrame): Pandas dataframe of images + """ + + return _flatten_column_to_dataframe(df_products, 'images') + + + +url = "https://yourshopifydomain.com" + +products = get_products(url) +products.to_csv('products.csv', index=False) +print('Products: count=', len(products)) + + +variants = get_variants(products) +variants.to_csv('variants.csv', index=False) +print('Variants: count=', len(variants)) + + +images = get_images(products) +images.to_csv('images.csv', index=False) +print('Images: count=', len(images)) From 4093a5598a4172a83c86c3dd725074210ce94e12 Mon Sep 17 00:00:00 2001 From: jdatjedd <130353431+jdatjedd@users.noreply.github.com> Date: Fri, 30 Aug 2024 11:03:23 +0530 Subject: [PATCH 03/12] Update all in one file --- all in one file | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/all in one file b/all in one file index 597fbbd..83961f2 100644 --- a/all in one file +++ b/all in one file @@ -1,3 +1,7 @@ +""" +1.copy the current code into the code editor of your choice to run python. +2.In line 122 You insert the url into the inverted commas and run the program with internet and the files will be created in yor folder + """ Shopify scraper Description: Scrapes products from a Shopify store by parsing products.json and converting it to a pandas DataFrame. From 417c1972286d576eca5a5b2608a09ea4895d7644 Mon Sep 17 00:00:00 2001 From: jdatjedd <130353431+jdatjedd@users.noreply.github.com> Date: Fri, 30 Aug 2024 11:11:10 +0530 Subject: [PATCH 04/12] Rename all in one file to all in one file.py --- all in one file => all in one file.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename all in one file => all in one file.py (100%) diff --git a/all in one file b/all in one file.py similarity index 100% rename from all in one file rename to all in one file.py From b8890a737b12772ffaca922f701b6bbb44283a6c Mon Sep 17 00:00:00 2001 From: AK <130353431+jdatjedd@users.noreply.github.com> Date: Fri, 12 Jun 2026 13:56:34 +0530 Subject: [PATCH 05/12] Revise README for Shopify Product Tracker Expanded README with detailed features, usage instructions, and project structure. --- README.md | 206 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 188 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 242125b..221b081 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,206 @@ -# ShopifyScraper +# šŸ›’ Shopify Product Tracker -ShopifyScraper is a Python package that scrapes data from Shopify. Unlike a regular web scraper, that needs to visit every page on a site, this package fetches Shopify's publicly visible `products.json` file, allowing you to scrape an entire store inventory in seconds. +A Python toolkit to fetch, monitor, and download product data from any public Shopify store — no API key required. -When the commands below are run, ShopifyScraper will extract the store inventory and save the products and product variants to Pandas dataframes, from where you can access or analyse the data or write it to CSV or database. +Track price changes, monitor inventory, and bulk-download product images all from the command line. -### Installation -To install ShopifyScraper, run the following command: +--- + +## ✨ Features + +- šŸ“‹ Fetch complete product listings (name, variant, price, inventory, images) +- šŸ’¾ Save everything to a clean, organised CSV +- šŸ“ˆ Track price changes on selected products across runs +- 🟢 Visual indicators for price drops, rises, and no change +- šŸ“ø Bulk download all product images into organised folders +- ā­ļø Smart skip — won't re-download images that already exist +- 🌐 Works on any public Shopify store without an API key + +--- + +## šŸ“ Project Structure + +``` +ShopifyTracker/ +│ +ā”œā”€ā”€ fetch.py # Fetches all products and saves to CSV +ā”œā”€ā”€ tracker.py # Checks current prices against saved prices +ā”œā”€ā”€ downloader.py # Downloads product images from the CSV +│ +ā”œā”€ā”€ products.csv # Auto-generated — full product list +ā”œā”€ā”€ tracked.csv # Auto-generated — your chosen tracked items +│ +└── product_images/ # Auto-generated — downloaded images + ā”œā”€ā”€ Product Name/ + │ └── Variant/ + │ ā”œā”€ā”€ image1.jpg + │ └── image2.jpg + └── ... +``` + +--- + +## āš™ļø Installation + +**Requirements:** Python 3.8+ ```bash -pip3 install git+https://github.com/practical-data-science/ShopifyScraper.git +pip install requests pandas ``` -### Usage +--- + +## šŸš€ How to Use + +### Step 1 — Fetch all products + +Set your store URL in `fetch.py`: ```python -from shopify_scraper import scraper +SHOP_URL = "https://yourstore.myshopify.com" +``` + +Then run: + +```bash +python fetch.py +``` -url = "https://yourshopifydomain.com" +This generates `products.csv` with every product listed on the store. -parents = scraper.get_products(url) -parents.to_csv('parents.csv', index=False) -print('Parents: ', len(parents)) +--- +### Step 2 — Track prices -children = scraper.get_variants(parents) -children.to_csv('children.csv', index=False) -print('Children: ', len(children)) +Open `products.csv` and note the **Row numbers** of items you want to monitor. +Set the same store URL in `tracker.py`, then run: -images = scraper.get_images(parents) -images.to_csv('images.csv', index=False) -print('Images: ', len(images)) +```bash +python tracker.py +``` +Enter your chosen row numbers when prompted: + +``` +Enter row numbers to track (from products.csv) +Example: 1 4 7 12 +Row numbers: 2 5 9 ``` +--- + +### Step 3 — Download images *(optional)* + +```bash +python downloader.py +``` + +Enter row numbers or type `all` to download every product's images: + +``` +Row numbers: all +``` + +Images are saved to `product_images/` organised by product and variant. + +--- + +## šŸ“Š Sample Output + +### fetch.py +``` +Fetching products... +Fetched page 1 — 250 products +Fetched page 2 — 143 products + +āœ… Saved 393 products to products.csv +šŸ“ø All photo URLs saved in the 'All Photos' column, separated by ' | ' +``` + +### tracker.py +``` +Tracking 3 item(s) — 2024-11-01 10:30:00 + +Row Product Saved Price Current Price Change +------------------------------------------------------------------------------ +2 Nike Air Max 90 ₹4999 ₹4499 🟢 DROP ₹500.00 +5 Adidas Ultraboost 22 ₹6999 ₹6999 ⚪ No change +9 Puma RS-X ₹3499 ₹3799 šŸ”“ RISE ₹300.00 +``` + +### downloader.py +``` +šŸ“¦ Row 2 — Nike Air Max 90 (Size 9): 3 image(s) + āœ… [1/3] Downloaded — image1.jpg + āœ… [2/3] Downloaded — image2.jpg + āœ… [3/3] Downloaded — image3.jpg + +────────────────────────────────────────── +āœ… Downloaded : 3 image(s) +āŒ Failed : 0 image(s) +šŸ“ Saved to : /ShopifyTracker/product_images +``` + +--- + +## šŸ—‚ļø CSV Format + +| Row | Product Name | Variant | Price | Inventory | Product ID | All Photos | +|-----|-------------|---------|-------|-----------|------------|------------| +| 1 | Nike Air Max | Size 9 | 4999 | 50 | 123456 | https://cdn/.../1.jpg \| https://cdn/.../2.jpg | +| 2 | Adidas Ultra | Size 10 | 6999 | 30 | 789012 | https://cdn/.../3.jpg | + +> **All Photos** column stores multiple image URLs separated by ` | ` +> Copy any URL and paste into your browser to view the image directly. + +--- + +## 🧠 Tech Stack + +| Tool | Purpose | +|------|---------| +| [Requests](https://docs.python-requests.org/) | Fetching data from Shopify's JSON API | +| [pandas](https://pandas.pydata.org/) | Reading and writing CSV files | +| [os / re](https://docs.python.org/3/library/os.html) | Folder creation and filename sanitisation | +| Shopify `/products.json` | Public product data endpoint | + +--- + +## āš™ļø Configuration + +| Variable | File | Default | Description | +|----------|------|---------|-------------| +| `SHOP_URL` | `fetch.py`, `tracker.py` | — | Target Shopify store URL | +| `CSV_FILE` | `downloader.py` | `products.csv` | Path to your product CSV | +| `OUTPUT_DIR` | `downloader.py` | `product_images` | Folder to save downloaded images | + +--- + +## āš ļø Known Limitations + +- Only works on Shopify stores that have **public product listings enabled**. Stores with password protection or restricted APIs will return a `403` or `404` error. +- Inventory quantity may show `N/A` if the store has hidden stock levels. +- Shopify's `/products.json` endpoint returns a maximum of **250 products per page** — the script handles pagination automatically. + +--- + +## šŸ’” Possible Improvements + +- šŸ“§ Email alerts when a tracked price drops +- šŸ“Š Price history graph using matplotlib +- šŸ—ƒļø SQLite database instead of CSV for larger stores +- 🌐 Flask web dashboard to manage tracked products +- ā° Scheduled auto-runs using the `schedule` library + +--- + +## šŸ“„ License + +This project is open source and available under the [MIT License](LICENSE). + +--- + +## šŸ¤ Contributing + +Pull requests are welcome! If you find a store where the scraper breaks or have an improvement in mind, feel free to open an issue. From 7f45ac3f8c342f6b62a6e0b90afe435613b5bcfa Mon Sep 17 00:00:00 2001 From: AK <130353431+jdatjedd@users.noreply.github.com> Date: Fri, 12 Jun 2026 13:58:56 +0530 Subject: [PATCH 06/12] Implement product fetching and CSV saving Fetches all products from a specified shop URL and saves them to a CSV file. --- Fetch.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 Fetch.py diff --git a/Fetch.py b/Fetch.py new file mode 100644 index 0000000..1a8c3df --- /dev/null +++ b/Fetch.py @@ -0,0 +1,64 @@ +import requests +import pandas as pd + +# ── CONFIG ──────────────────────────────────────────────────────────────────── +SHOP_URL = "https://www.peepultree.world" # Replace with target store URL + +def fetch_all_products(shop_url): + products_list = [] + page = 1 + + while True: + url = f"{shop_url}/products.json?limit=250&page={page}" + response = requests.get(url) + + if response.status_code != 200: + print(f"Failed to fetch page {page}: {response.status_code}") + break + + data = response.json().get("products", []) + + if not data: + break + + for product in data: + title = product["title"] + + # ── Collect ALL image URLs for this product ──────────────────── + images = product.get("images", []) + all_photo_urls = " | ".join([img["src"] for img in images]) + # URLs are separated by " | " so the CSV stays readable + # Example: "https://cdn.../img1.jpg | https://cdn.../img2.jpg" + + if not all_photo_urls: + all_photo_urls = "No images" + + # ── Each variant gets its own row ────────────────────────────── + for variant in product["variants"]: + products_list.append({ + "Product Name" : title, + "Variant" : variant["title"], + "Price" : float(variant["price"]), + "Inventory" : variant.get("inventory_quantity", "N/A"), + "Product ID" : variant["id"], + "All Photos" : all_photo_urls # All URLs in one cell + }) + + print(f"Fetched page {page} — {len(data)} products") + page += 1 + + return products_list + + +def save_to_csv(products_list): + df = pd.DataFrame(products_list) + df.index += 1 + df.to_csv("products.csv", index_label="Row") + print(f"\nāœ… Saved {len(products_list)} products to products.csv") + print("šŸ“ø All photo URLs saved in the 'All Photos' column, separated by ' | '") + + +if __name__ == "__main__": + print("Fetching products...") + products = fetch_all_products(SHOP_URL) + save_to_csv(products) From 07b3584bba386948c587d879a260414230b12b81 Mon Sep 17 00:00:00 2001 From: AK <130353431+jdatjedd@users.noreply.github.com> Date: Fri, 12 Jun 2026 14:00:21 +0530 Subject: [PATCH 07/12] Add files via upload --- Tracker.py | 66 ++++++++++++++++++++++++++++++ downloader.py | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 Tracker.py create mode 100644 downloader.py diff --git a/Tracker.py b/Tracker.py new file mode 100644 index 0000000..96d1d14 --- /dev/null +++ b/Tracker.py @@ -0,0 +1,66 @@ +import requests +import pandas as pd +from datetime import datetime + +SHOP_URL = "https://yourstore.myshopify.com" # Same store URL as fetch.py + +def get_current_price(product_id): + """ + Fetches the latest price of a variant directly from Shopify's API + using the product variant ID saved in the CSV. + """ + url = f"{SHOP_URL}/products.json?limit=250" + response = requests.get(url) + + if response.status_code != 200: + return None + + for product in response.json().get("products", []): + for variant in product["variants"]: + if variant["id"] == product_id: + return float(variant["price"]) + return None + + +def track_prices(row_numbers): + """ + Loads products.csv, filters chosen rows, checks current prices, + and reports any changes. + """ + df = pd.read_csv("products.csv", index_col="Row") + + chosen = df.loc[row_numbers] # Filter only chosen rows + + # Save chosen items to tracked.csv on first run + chosen.to_csv("tracked.csv") + print(f"\nTracking {len(chosen)} item(s) — {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") + print(f"{'Row':<5} {'Product':<40} {'Saved Price':<15} {'Current Price':<15} {'Change'}") + print("-" * 90) + + for row, item in chosen.iterrows(): + saved_price = item["Price"] + product_id = item["Product ID"] + current_price = get_current_price(int(product_id)) + + if current_price is None: + print(f"{row:<5} {item['Product Name']:<40} ₹{saved_price:<14} Could not fetch") + continue + + # Calculate change + change = current_price - saved_price + if change < 0: + status = f"🟢 DROP ₹{abs(change):.2f}" + elif change > 0: + status = f"šŸ”“ RISE ₹{change:.2f}" + else: + status = "⚪ No change" + + print(f"{row:<5} {item['Product Name']:<40} ₹{saved_price:<14} ₹{current_price:<14} {status}") + + +if __name__ == "__main__": + print("Enter row numbers to track (from products.csv)") + print("Example: 1 4 7 12") + user_input = input("Row numbers: ").strip().split() + row_numbers = [int(r) for r in user_input] + track_prices(row_numbers) diff --git a/downloader.py b/downloader.py new file mode 100644 index 0000000..7dc903f --- /dev/null +++ b/downloader.py @@ -0,0 +1,108 @@ +import requests +import pandas as pd +import os +import re + +# ── CONFIG ──────────────────────────────────────────────────────────────────── +CSV_FILE = "products.csv" # Your product CSV +OUTPUT_DIR = "product_images" # Folder where images will be saved + +def sanitise_name(name): + """ + Removes special characters from product names so they + can be safely used as folder names on any OS. + """ + return re.sub(r'[\\/*?:"<>|]', "", name).strip() + + +def download_images_for_rows(row_numbers): + """ + Reads the CSV, filters chosen rows, and downloads + all photos for each product into its own subfolder. + """ + df = pd.read_csv(CSV_FILE, index_col="Row") + + # Validate row numbers + valid_rows = [r for r in row_numbers if r in df.index] + invalid_rows = [r for r in row_numbers if r not in df.index] + + if invalid_rows: + print(f"āš ļø Skipping invalid rows: {invalid_rows}") + + if not valid_rows: + print("āŒ No valid rows found. Exiting.") + return + + chosen = df.loc[valid_rows] + + # Create main output directory + os.makedirs(OUTPUT_DIR, exist_ok=True) + + total_downloaded = 0 + total_failed = 0 + + for row, item in chosen.iterrows(): + product_name = sanitise_name(item["Product Name"]) + variant = sanitise_name(str(item["Variant"])) + photo_urls = item["All Photos"] + + if pd.isna(photo_urls) or photo_urls == "No images": + print(f"\nāš ļø Row {row} — {product_name} ({variant}): No images found, skipping.") + continue + + # Split the " | " separated URLs back into a list + urls = [url.strip() for url in photo_urls.split("|")] + + # Create a subfolder per product variant + # e.g. product_images/Nike Air Max/Size 9/ + folder = os.path.join(OUTPUT_DIR, product_name, variant) + os.makedirs(folder, exist_ok=True) + + print(f"\nšŸ“¦ Row {row} — {product_name} ({variant}): {len(urls)} image(s)") + + for i, url in enumerate(urls, start=1): + # Derive a clean filename from the URL + # e.g. https://cdn.shopify.com/.../image.jpg → image.jpg + filename = url.split("?")[0].split("/")[-1] # Strip query params + save_path = os.path.join(folder, filename) + + # Skip if already downloaded + if os.path.exists(save_path): + print(f" ā­ļø [{i}/{len(urls)}] Already exists — {filename}") + continue + + try: + response = requests.get(url, timeout=10) + if response.status_code == 200: + with open(save_path, "wb") as f: + f.write(response.content) + print(f" āœ… [{i}/{len(urls)}] Downloaded — {filename}") + total_downloaded += 1 + else: + print(f" āŒ [{i}/{len(urls)}] Failed (HTTP {response.status_code}) — {url}") + total_failed += 1 + except requests.exceptions.RequestException as e: + print(f" āŒ [{i}/{len(urls)}] Error — {e}") + total_failed += 1 + + # ── Summary ─────────────────────────────────────────────────────────────── + print("\n" + "─" * 50) + print(f"āœ… Downloaded : {total_downloaded} image(s)") + print(f"āŒ Failed : {total_failed} image(s)") + print(f"šŸ“ Saved to : {os.path.abspath(OUTPUT_DIR)}") + + +if __name__ == "__main__": + print("Enter row numbers to download images for (from products.csv)") + print("Example: 1 4 7 12") + print("Type 'all' to download images for every product\n") + + user_input = input("Row numbers: ").strip() + + if user_input.lower() == "all": + df = pd.read_csv(CSV_FILE, index_col="Row") + row_numbers = list(df.index) + else: + row_numbers = [int(r) for r in user_input.split()] + + download_images_for_rows(row_numbers) From b9f157b58e5f0b74eb2829b02529077e29d2fe0f Mon Sep 17 00:00:00 2001 From: AK <130353431+jdatjedd@users.noreply.github.com> Date: Fri, 12 Jun 2026 14:03:17 +0530 Subject: [PATCH 08/12] Delete setup.cfg --- setup.cfg | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b88034e..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -description-file = README.md From a4f8908f5655b499c9007e243c52c93532a53195 Mon Sep 17 00:00:00 2001 From: AK <130353431+jdatjedd@users.noreply.github.com> Date: Fri, 12 Jun 2026 14:03:38 +0530 Subject: [PATCH 09/12] Delete setup.py --- setup.py | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index b3ac180..0000000 --- a/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -from setuptools import setup - -from os import path -this_directory = path.abspath(path.dirname(__file__)) -with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: - long_description = f.read() - -setup( - name='ShopifyScraper', - packages=['shopify_scraper'], - version='0.002', - license='MIT', - description='ShopifyScraper is a Python package that scrapes Shopify products and variants to Pandas dataframes.', - long_description=long_description, - long_description_content_type='text/markdown', - author='Matt Clarke', - author_email='matt@practicaldatascience.co.uk', - url='https://github.com/practical-data-science/ShopifyScraper', - download_url='https://github.com/practical-data-science/ShopifyScraper/archive/master.zip', - keywords=['python', 'requests', 'pandas', 'shopify', 'scraping'], - classifiers=[ - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3.8', - ], - install_requires=['pandas', 'requests'] -) \ No newline at end of file From 65dab0439d5a823467c05820fafb173425db6793 Mon Sep 17 00:00:00 2001 From: AK <130353431+jdatjedd@users.noreply.github.com> Date: Fri, 12 Jun 2026 14:03:46 +0530 Subject: [PATCH 10/12] Delete requirements.txt --- requirements.txt | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index a94cf69..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests -pandas \ No newline at end of file From 5109296e0b03534bc95dd0cff10403319633b3e0 Mon Sep 17 00:00:00 2001 From: AK <130353431+jdatjedd@users.noreply.github.com> Date: Fri, 12 Jun 2026 14:03:56 +0530 Subject: [PATCH 11/12] Delete all in one file.py --- all in one file.py | 136 --------------------------------------------- 1 file changed, 136 deletions(-) delete mode 100644 all in one file.py diff --git a/all in one file.py b/all in one file.py deleted file mode 100644 index 83961f2..0000000 --- a/all in one file.py +++ /dev/null @@ -1,136 +0,0 @@ -""" -1.copy the current code into the code editor of your choice to run python. -2.In line 122 You insert the url into the inverted commas and run the program with internet and the files will be created in yor folder - -""" -Shopify scraper -Description: Scrapes products from a Shopify store by parsing products.json and converting it to a pandas DataFrame. -Author: Matt Clarke -""" - -import json -import pandas as pd -import requests - -def get_json(url: str, page: int) -> str: - """ - Get Shopify products.json from a store URL. - - Args: - url (str): URL of the store. - page (int): Page number of the products.json. - Returns: - products_json: Products.json from the store. - """ - - response = requests.get(f'{url}/products.json?limit=250&page={page}', timeout=5) - products_json = response.text - response.raise_for_status() - return products_json - -def _products_json_to_df(products_json: str) -> pd.DataFrame: - """ - Convert products.json to a pandas DataFrame. - - Args: - products_json (json): Products.json from the store. - Returns: - df: Pandas DataFrame of the products.json. - """ - - products_dict = json.loads(products_json) - df = pd.DataFrame.from_dict(products_dict['products']) - return df - -def get_products(url: str) -> pd.DataFrame: - """ - Get all products from a store. - - Returns: - df: Pandas DataFrame of the products.json. - """ - - results = True - page = 1 - df = pd.DataFrame() - - while results: - products_json = get_json(url, page) - products_dict = _products_json_to_df(products_json) - - if len(products_dict) == 0: - break - else: - df = pd.concat([df, products_dict], ignore_index=True) - page += 1 - - df['url'] = f"{url}/products/" + df['handle'] - return df - -def get_variants(products: pd.DataFrame) -> pd.DataFrame: - """Get variants from a table of products. - - Args: - products (pd.DataFrame): Pandas dataframe of products from get_products() - - Returns: - variants (pd.DataFrame): Pandas dataframe of variants - """ - - products['id'].astype(int) - df_variants = pd.DataFrame() - - for row in products.itertuples(index='True'): - for variant in getattr(row, 'variants'): - df_variants = pd.concat([df_variants, pd.DataFrame.from_records(variant, index=[0])]) - - df_variants['id'].astype(int) - df_variants['product_id'].astype(int) - df_product_data = products[['id', 'title', 'vendor']] - df_product_data = df_product_data.rename(columns={'title': 'product_title', 'id': 'product_id', 'vendor': 'product_vendor'}) - df_variants = df_variants.merge(df_product_data, left_on='product_id', right_on='product_id', validate='m:1') - return df_variants - -def _flatten_column_to_dataframe(df: pd.DataFrame, col: str) -> pd.DataFrame: - """Return a Pandas dataframe based on a column that contains a list of JSON objects. - - Args: - df (Pandas dataframe): The dataframe to be flattened. - col (str): The name of the column that contains the JSON objects. - - Returns: - Pandas dataframe: A new dataframe with the JSON objects expanded into columns. - """ - - rows = [item for row in df[col] for item in row] - return pd.DataFrame(rows) - -def get_images(df_products: pd.DataFrame) -> pd.DataFrame: - """Get images from a list of products. - - Args: - df_products (pd.DataFrame): Pandas dataframe of products from get_products() - - Returns: - images (pd.DataFrame): Pandas dataframe of images - """ - - return _flatten_column_to_dataframe(df_products, 'images') - - - -url = "https://yourshopifydomain.com" - -products = get_products(url) -products.to_csv('products.csv', index=False) -print('Products: count=', len(products)) - - -variants = get_variants(products) -variants.to_csv('variants.csv', index=False) -print('Variants: count=', len(variants)) - - -images = get_images(products) -images.to_csv('images.csv', index=False) -print('Images: count=', len(images)) From 2403884adb07077d2f9b9c89a21e42418edfc06e Mon Sep 17 00:00:00 2001 From: AK <130353431+jdatjedd@users.noreply.github.com> Date: Fri, 12 Jun 2026 14:04:04 +0530 Subject: [PATCH 12/12] Delete .gitignore --- .gitignore | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index fe1f6e5..0000000 --- a/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -venv -.idea -example.py -parents.csv -children.csv -images.csv \ No newline at end of file