Merge pull request #23 from JustInternetAI/Grace

Updating development with new docker compose file

Author: graceannmad

.gitignore

@@ -10,3 +10,6 @@ feed*.json
 *.pyc
 *.pyo
 *.pyd
+
+#Ignore saved data
+data/*

bbc_world.xml

@@ -28,9 +28,35 @@ services:
       dockerfile: .docker/Dockerfile
     volumes:
       - .:/workspace:cached
+      - mongo_data:/data/db 
     depends_on:
       - redis
     #entrypoint: ["/usr/local/bin/entrypoint.sh"]
     command: ["celery", "-A", justinsight.celery, "worker", "--loglevel=info"] #the lowercase j is actually so important
 
+  celery_beat:
+    build:
+      context: .
+      dockerfile: .docker/Dockerfile
+    volumes:
+      - .:/workspace:cached
+    depends_on:
+      - redis
+    #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+    command: ["celery", "-A", justinsight.celery, "beat", "--loglevel=info"] #the lowercase j is actually so important
+
+  mongo:
+    image: mongo:latest
+    container_name: mongo
+    restart: unless-stopped
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: myuser
+      MONGO_INITDB_ROOT_PASSWORD: mypassword
+    ports:
+      - "27017:27017"
+    volumes:
+      - mongo_data:/data/db 
+      - .:/workspace:cached
 
+volumes:
+  mongo_data:

docker-compose.yml

@@ -1 +1,6 @@
-celery[redis]
+beautifulsoup4
+celery[redis]
+feedparser
+playwright
+pymongo
+requests

requirements.txt

@@ -10,15 +10,23 @@
 
 # Optional beat schedule
 app.conf.beat_schedule = {
-    "sample-task-every-10-seconds": {
-        "task": "justinsight.tasks.sample_task",
-        "schedule": 10.0,
+    # "sample-task-every-5-seconds": {
+    #     "task": "justinsight.tasks.sample_task",
+    #     "schedule": 5.0,
+    #     "args": (),
+    # },
+
+    "check-BBCfeed-every-5-minutes": {
+        "task": "justinsight.tasks.bbcLogger_task",
+        "schedule": 300.0,
         "args": (),
     },
-        "sample-task-every-5-seconds": {
-        "task": "justinsight.tasks.sample_task",
-        "schedule": 5.0,
+
+    "check-NYTfeed-every-5-minutes": {
+        "task": "justinsight.tasks.nytLogger_task",
+        "schedule": 300.0,
         "args": (),
     },
+
     #schedule more tasks here
 }

src/justinsight/celery.py

@@ -1,11 +1,22 @@
 # tasks.py
 from celery import shared_task
-from ingest.bbc_rss import testTask
+from ingest.bbc_rss import check_and_save_new_entries as check_and_save_bbc
+from ingest.nyt_rss import check_and_save_new_entries as check_and_save_nyt
 
 
 @shared_task
 def sample_task():
-    testTask()
+    print("Sample task!")
     return "done!"
 
+@shared_task
+def bbcLogger_task():
+    check_and_save_bbc()
+    return "BBC RSS Feed checked."
+
+@shared_task
+def nytLogger_task():
+    check_and_save_nyt()
+    return "NYT RSS Feed checked."
+
 #Add more tasks here in the format of the one above