Initial backend

README.md

@@ -14,3 +14,44 @@ CanadianStartupJobs is built to strengthen Canada’s startup ecosystem by makin
 - Verified Canadian-owned startups only  
 - Simple job posting and management dashboard for founders
 - Public API for regional innovation hubs and directories, such as university accelerators
+
+## Development Setup
+
+### Prerequisites
+- Docker and Docker Compose
+- Node.js (for backend services)
+
+### Database Setup
+
+Start PostgreSQL using Docker Compose:
+
+```bash
+docker-compose up -d postgres
+```
+
+This will start a PostgreSQL 16 database with the following default configuration:
+- **User**: `postgres`
+- **Password**: `postgres`
+- **Database**: `canadian_startup_jobs`
+- **Port**: `5432`
+
+You can customize these settings by creating a `.env` file in the project root with:
+
+```env
+POSTGRES_USER=your_user
+POSTGRES_PASSWORD=your_password
+POSTGRES_DB=your_database
+POSTGRES_PORT=5432
+```
+
+The database data will be persisted in a Docker volume. To stop the database:
+
+```bash
+docker-compose down
+```
+
+To remove the database and all data:
+
+```bash
+docker-compose down -v
+```

backend/resourceList.md

@@ -0,0 +1,166 @@
+## To Contact:
+Startup Blink has the list and tons of other data (https://www.startupblink.com/startup-ecosystem/canada)
+IRAP or IPON
+
+## Candidate Management System Links:
+https://jobs.ashbyhq.com/<company>
+https://<company>.applytojob.com
+https://jobs.lever.co/<company>
+
+## Company Careers Boards:
+- https://<companyURL>/careers
+- https://<companyURL>/jobs
+- https://<companyURL>/en-ca/careers
+- https://<companyURL>/en-ca/jobs
+
+## Master Lists: 
+- https://www.bycanada.tech/
+- https://www.canstarthub.ca/discover
+- https://canadastartups.co/
+- https://growthlist.co/canada-startups/
+
+## Smaller Programs Lists: 
+- https://drive.google.com/file/d/1Nx-0lyb_A24L6fJcHQm3AEnR_JFPE0OH/view
+- https://drive.google.com/file/d/1lRe_ZtmlYzt89g_RAViWaBvFZG6tEJSU/view?pli=1 
+- https://drive.google.com/file/d/110rHwUA_I0q4uUevgF46FBhyQ1t3uYuV/view
+- https://drive.google.com/file/d/1JARpTeCVg4qjEhWMLUn8yhf9a8bQNMMn/view?usp=share_link 
+- https://cswaccelerator.com/cohort-2-2/ 
+- https://cswaccelerator.com/cohort1/
+- https://innovationfactory.ca/clients/
+- https://directory.nextcanada.com/directory/ventures/
+- https://startup.google.com/alumni/directory/?_gl=1*utxxg3*_up*MQ..*_ga*NTE2NDcyNDU3LjE3NjMwMTY1OTk.*_ga_GCB35PQ9X3*czE3NjMwMTY1OTgkbzEkZzAkdDE3NjMwMTY2MTQkajQ0JGwwJGgw&region=canada
+- https://velocityincubator.com/companies/
+- https://www.intuit.com/ca/prosperity-accelerator/alumni/ https://www.safexconnected.com/cohorts 
+- https://oneeleven-ca.squarespace.com/alumni // https://oneeleven-ca.squarespace.com/members
+- https://dmz.torontomu.ca/startup-directory
+- https://500.co/portfolio
+- https://www1.communitech.ca/companies
+- https://www.mcgill.ca/dobson/ourstartups
+- https://www.ualberta.ca/en/medicine/research/health-innovation-hub/companies.html
+- https://www.launchacademy.ca/alumni/
+- https://creativedestructionlab.com/companies/
+- https://www.founderfuel.com/companies/
+- https://www.yorku.ca/yspace/startups/
+- https://www.antler.co/portfolio
+- https://entrepreneurs.utoronto.ca/our-startups/startups-directory/
+- https://innovation.ubc.ca/entrepreneurship-ventures/portfolio-companies
+- https://www.acceleratorcentre.com/our-startups
+- https://www.l-spark.com/meet-our-companies/
+- https://airtable.com/app1Z7w2CdfNJt9rB/shrM8IADuuZf3wvOU/tblxzzU3yUX1ru667
+- https://hatchery.engineering.utoronto.ca/our-startup-teams/
+- https://www.tandemlaunch.com/portfolio
+- https://www.queensu.ca/innovationcentre/community/meet-startups
+- https://www.investottawa.ca/io-accelerator-companies/
+- https://entrepreneurship.uwo.ca/accelerator/our-venture-directory/
+- https://platformcalgary.webflow.io/community?tab=partners
+- https://icics.ubc.ca/hatch/alumni-hatch-ventures/
+- https://airtable.com/appurLNyQbHbSsCbJ/shrsOFwAHmVudJvsG/tbllsMtZKORBUYbXc
+- https://www.launchlab.ca/client-stories
+- https://edmontonunlimited.com/alumni-companies/
+- https://h2i.utoronto.ca/ventures/
+- https://foresightcac.com/companies
+- https://www.citm.ca/clients/
+- https://www.torontomu.ca/zone-learning/legal-innovation-zone/startups/ 
+- https://www.torontomu.ca/zone-learning/legal-innovation-zone/startups/#!tab-1745414258423-alumni-startups
+- https://www.torontomu.ca/zone-learning/biomedical-zone/startups1/
+- https://jnjinnovation.com/JLABSNavigator/
+- https://venturelabs.ca/companies/
+- https://www.ualberta.ca/en/business/alumni/business-directory.html
+- https://spinup.utm.utoronto.ca/our-startups/
+- https://app.marsdd.com/directory
+- https://mtlab.ca/startups/
+- https://lassonde.yorku.ca/best/startups/
+- https://www.district3.co/stories
+- https://www.georgebrown.ca/startgbc/alumni-entrepreneurs
+- https://www.plugandplaytechcenter.com/venture-capital/startup-portfolio
+- https://www.torontomu.ca/zone-learning/sdz/startups/
+- https://www.torontomu.ca/zone-learning/fashion-zone/companies/
+- https://entrepreneurs.utoronto.ca/our-startups/startups-directory/
+- https://holtxchange.com/portfolio/
+- https://www.stationfintech.com/en/our-startups
+- https://www.co-labs.ca/our-startups
+- https://www.loi.vc/portfolio/
+- https://spring.is/impact-capital/portfolio/
+- https://www.torontomu.ca/svz/startups/
+
+## VC Portfolios:
+- 
+
+Transmedia Zone (TMU) — Startups: https://www.torontomu.ca/transmedia-zone/startups/
+Startup Calgary — Showcase/Alumni (closest equivalent): https://www.startupcalgary.ca/launch-party
+New Ventures BC — Alumni/Companies: https://www.newventuresbc.com/alumni
+Global Startups — Portfolio: https://www.globalstartups.io/portfolio
+Brampton Venture Zone (TMU) — Companies: https://www.bramptonventurezone.ca/companies
+Ideas Inc (Saskatoon) — Companies: https://ideasinc.ca/companies/
+Foresight Canada — Ventures: https://foresightcac.com/ventures/
+Brampton Entrepreneur Centre — Homepage fallback: https://www.brampton.ca/EN/Business/BEC/Pages/Welcome.aspx
+Parkdale Centre for Innovation — Companies/Portfolio: https://www.parkdaleinnovates.org/
+Centre for Entrepreneurship (U of T) — Homepage fallback (please clarify the specific unit)
+Canada’s Tech Network — Members (closest equivalent): https://canadastechnetwork.ca/members/
+Digital Media and Gaming Incubator — Homepage fallback (please clarify which institution)
+Startup Garage (uOttawa/Invest Ottawa) — Portfolio/Alumni: https://www.investottawa.ca/programs/startup-garage/ (closest equivalent)
+Lazaridis ScaleUp Program — Companies: https://lazaridisinstitute.ca/scaleup/companies/ (if moved, main program page: https://lazaridisinstitute.ca/scaleup/)
+Clean Energy Zone (TMU) — Companies/Startups: https://www.torontomu.ca/clean-energy-zone/
+ideaHUB — Homepage fallback (please clarify institution, e.g., Dalhousie vs. other)
+Halton Region Small Business Centre — Homepage fallback: https://www.halton.ca/For-Business/Starting-a-Business/Small-Business-Centre
+Centech (Montréal) — Startups: https://centech.co/startups/
+TechAlliance (Southwestern Ontario) — Client Directory: https://techalliance.ca/directory/ (closest equivalent)
+CEED (Halifax) — Client stories/ventures (closest equivalent): https://www.ceed.ca/
+LiftOff — Homepage fallback (please clarify which program/org)
+Design Fabrication Zone (TMU) — Startups: https://www.torontomu.ca/design-fabrication-zone/startups/
+The Forge (McMaster) — Companies: https://theforgehamilton.ca/companies/
+Innovation Boost Zone (TMU) — Startups: https://www.torontomu.ca/innovation-boost-zone/startups/
+La Piscine MTL — Startups/Portfolio: https://lapiscine.co/ (portfolio within site)
+Canada’s Music Incubator — Programs/clients (closest equivalent): https://canadasmusicincubator.com/
+ACCEL (Centennial College) — Homepage fallback: https://www.centennialcollege.ca/centres-institutes/accel
+Artscape Daniels Launchpad — Members/Residents (closest equivalent): https://www.artscape.ca/launchpad/
+Shad Canada — Alumni/success stories (closest equivalent): https://www.shad.ca/
+Canadian Technology Accelerators (TCS) — Programs/companies (closest equivalent): https://www.tradecommissioner.gc.ca/cta-atc/index.aspx?lang=eng
+La base (HEC Montréal) — Startups/Portfolio: https://www.hec.ca/en/entrepreneurs/la-base/
+C100 — Fellows/Community (closest equivalent): https://www.thec100.org/fellows
+North Forge — Clients/Startups: https://northforge.ca/clients/
+Altitude Accelerator — Ventures: https://altitudeaccelerator.ca/ventures/
+Brampton Venture Zone — Companies: https://www.bramptonventurezone.ca/companies
+Rogers Cybersecure Catalyst — Accelerator Companies: https://www.torontomu.ca/cybersecure-catalyst/accelerator/companies/
+Brampton Entrepreneur Centre — Homepage: https://www.brampton.ca/EN/Business/BEC/Pages/Welcome.aspx
+Bhive Brampton — Companies: https://www.bhive.ca/companies
+TechPlace (Burlington) — Residents: https://techplace.ca/residents/
+Halton Region Small Business Centre — Homepage: https://www.halton.ca/For-Business/Starting-a-Business/Small-Business-Centre
+Innovation Factory — Client Directory: https://innovationfactory.ca/client-directory/
+Alberta Catalyzer — Program page: https://www.albertacatalyzer.com/
+Plug and Play Alberta — Program page: https://www.plugandplaytechcenter.com/alberta/
+Alberta IoT Association — Directory: https://www.albertaiot.com/directory/
+Pacific Technology Ventures — Homepage (ambiguous): https://www.pacifictechnologyventures.com/ (please confirm)
+Intrinsic Innovations — Homepage: https://www.intrinsicinnovations.ca/
+Creative Destruction Lab Calgary — Companies (global listing): https://creativedestructionlab.com/companies/
+REACH Canada — Program: https://www.reachinsurtech.com/canada
+Foresight Canada — Ventures: https://foresightcac.com/ventures/
+Startup Calgary — Launch Party alumni: https://www.startupcalgary.ca/launch-party
+Platform Calgary — Homepage: https://www.platformcalgary.com/
+C100 — Fellows/Community: https://www.thec100.org/fellows
+
+https://www.foundersbeta.com/startup-directory/categories/edmonton-startup-accelerators-incubators/
+https://www.foundersbeta.com/startup-directory/categories/guelph-startup-accelerators-incubators/
+https://www.foundersbeta.com/startup-directory/categories/halifax-startup-accelerators-incubators/
+https://www.foundersbeta.com/startup-directory/categories/hamilton-startup-accelerators-incubators/
+https://www.foundersbeta.com/startup-directory/categories/kingston-startup-accelerators-incubators/
+https://www.foundersbeta.com/startup-directory/categories/london-ontario-startup-accelerators-incubators/
+https://www.foundersbeta.com/startup-directory/categories/markham-startup-accelerators-incubators/
+https://www.foundersbeta.com/startup-directory/categories/mississauga-startup-accelerators-incubators/
+https://www.foundersbeta.com/startup-directory/categories/montreal-startup-accelerators-incubators/
+https://www.foundersbeta.com/startup-directory/categories/ottawa-startup-accelerators-incubators/
+https://www.foundersbeta.com/startup-directory/categories/startup-accelerators-and-incubators/
+https://www.foundersbeta.com/startup-directory/categories/toronto-startup-accelerators-incubators/
+https://www.foundersbeta.com/startup-directory/categories/vancouver-startup-accelerators-incubators/
+https://www.foundersbeta.com/startup-directory/categories/waterloo-startup-accelerators-incubators/
+
+# Other Startup Resources
+- eCommerce North: https://www.ecommercenorth.ca/
+- Volta Effect: https://www.voltaeffect.com/
+- Alberta Innovates: https://albertainnovates.ca/
+- TKSociety: https://www.theksociety.com/
+- Altitude Accelerator: https://altitudeaccelerator.ca/ventures/
+- TMU Cybersecure: https://cybersecurecatalyst.ca/catalyst-cyber-accelerator/
+- Bioenterprise Canada: https://bioenterprise.ca/member-directory/
+- Toronto Business Development Centre: https://tbdc.com/
+- The Forum: https://www.theforum.ca/

backend/scraper-cron/.env.example

@@ -0,0 +1,3 @@
+FIRE_CRAWL_API_KEY=
+NODE_ENV=development
+OPENAI_API_KEY=

backend/scraper-cron/README.md

@@ -0,0 +1,35 @@
+# Scraper Cron
+
+Service to scrape data from data sources as listed in resourceList.md
+
+## How It Works
+
+Scrapes company directories from `sources.ts`,  discovers job board URLs, then batch scrapes them using Firecrawl with a structured job schema. Results are written to `new_jobs.json`.
+
+## Uses
+
+Used a mixture of playwright, open ai, and firecrawl to accomplish this. Firecrawl isn't great at pagination so we opted to use playwright + open ai for the heavy lifting.
+
+## Environment Variables
+
+Copy `.env.example` to `.env` and fill in:
+- `FIRE_CRAWL_API_KEY` - Firecrawl API key for web scraping
+- `OPENAI_API_KEY` - OpenAI API key for AI extraction
+- `REDIS_URL` - Redis connection URL (or use `REDIS_HOST` and `REDIS_PORT`)
+
+## Startup
+
+```bash
+npm install
+npm run dev
+```
+
+Scrapes job boards from company directories and outputs to `new_jobs.json`.
+
+
+
+    parallelizaton/queueing
+    setting this up to run on a cron
+    working onfault tolerance
+    saving to jobs to the db
+    filtering jobs/companies being parsed - probably most important to make sure they are actually startups.

backend/scraper-cron/bull-board-server.ts

@@ -0,0 +1,43 @@
+import express from "express";
+import { createBullBoard } from "@bull-board/api";
+import { BullMQAdapter } from "@bull-board/api/bullMQAdapter";
+import { ExpressAdapter } from "@bull-board/express";
+import {
+  mapCompanyDirQueue,
+  companyDirectoryQueue,
+  jobBoardQueue,
+} from "./queues";
+import dotenv from "dotenv";
+
+dotenv.config();
+
+const app = express();
+
+// Create Express adapter for Bull Board
+const serverAdapter = new ExpressAdapter();
+serverAdapter.setBasePath("/admin/queues");
+
+// Create BullMQ Board with all queues
+createBullBoard({
+  queues: [
+    new BullMQAdapter(mapCompanyDirQueue),
+    new BullMQAdapter(companyDirectoryQueue),
+    new BullMQAdapter(jobBoardQueue),
+  ],
+  serverAdapter,
+});
+
+// Mount BullMQ Board at the '/admin/queues' route
+app.use("/admin/queues", serverAdapter.getRouter());
+
+// Health check endpoint
+app.get("/", (req, res) => {
+  res.redirect("/admin/queues");
+});
+
+const PORT = process.env.BULL_BOARD_PORT || 3000;
+
+app.listen(PORT, () => {
+  // BullMQ Board running
+});
+

backend/scraper-cron/clear-queues.ts

@@ -0,0 +1,49 @@
+// Simple script to clear all BullMQ queues
+import {
+  mapCompanyDirQueue,
+  companyDirectoryQueue,
+  jobBoardQueue,
+  closeAllQueues,
+} from "./queues";
+import { connectRedis } from "./redisClient";
+import dotenv from "dotenv";
+
+dotenv.config();
+
+const clearQueues = async () => {
+  await connectRedis();
+
+  const queues = [
+    { name: "map-company-directories", queue: mapCompanyDirQueue },
+    { name: "company-directories", queue: companyDirectoryQueue },
+    { name: "job-boards", queue: jobBoardQueue },
+  ];
+
+  for (const { name, queue } of queues) {
+    try {
+      // Get counts before clearing
+      const waiting = await queue.getWaitingCount();
+      const active = await queue.getActiveCount();
+      const completed = await queue.getCompletedCount();
+      const failed = await queue.getFailedCount();
+      const delayed = await queue.getDelayedCount();
+
+      const total = waiting + active + completed + failed + delayed;
+
+      if (total > 0) {
+        // Clear all job states
+        await queue.obliterate({ force: true });
+      }
+    } catch (error) {
+      // Error clearing queue
+    }
+  }
+
+  await closeAllQueues();
+  process.exit(0);
+};
+
+clearQueues().catch((error) => {
+  process.exit(1);
+});
+