diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..249672e9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +tasks.log +.vscode/* +.env +.venv/* diff --git a/__pycache__/tasks.cpython-37.pyc b/__pycache__/tasks.cpython-37.pyc new file mode 100644 index 00000000..d883748a Binary files /dev/null and b/__pycache__/tasks.cpython-37.pyc differ diff --git a/jobs/07d9860141d945e2a168eeff1b7cca82.json b/jobs/07d9860141d945e2a168eeff1b7cca82.json new file mode 100644 index 00000000..494fc48a --- /dev/null +++ b/jobs/07d9860141d945e2a168eeff1b7cca82.json @@ -0,0 +1 @@ +{"reservationId": "640fd9793ce80b003546d5a5", "conversation": {"meta": {"reservations": [{"_id": "640fd9793ce80b003546d5a5", "checkIn": "2023-08-03T19:00:00.000Z", "checkOut": "2023-08-05T15:00:00.000Z"}], "guestName": "Mehdi"}, "guestId": "63fa63f22c3ab9002b42668c", "language": "en", "lastUpdatedFromGuest": "2023-03-17T00:00:42.000Z", "isRead": true, "snoozedUntil": null, "status": "ARCHIVED", "integration": {"platform": "airbnb2", "_id": "640fc8a4688b4a002d46b7b9", "airbnb2": {"id": "1428096168", "guestId": 302389}}, "accountId": "63f9222c980d05002c4e554d", "createdAt": "2023-04-29T03:36:28.000Z", "lastUpdatedAt": "2023-03-24T00:17:41.395Z", "lastModifiedAt": "2023-04-27T15:18:39.093Z", "conversationWith": "Guest", "priority": 10, "pendingTasks": [], "firstReceptionist": "Mohamed El Hadi Msaid", "_id": "640fd9794f14fc002f7d238d", "thread": [{"_id": "644c90c2e5b07e0034c31219", "module": "airbnb2", "type": "fromGuest", "body": "Hi Seb, \nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?", "createdAt": "2023-04-29T03:36:28.000Z", "cc": [], "bcc": [], "postId": "644c90c2e5b07e0034c31219", "to": [], "feedback": {}}], "module": "airbnb2", "type": "fromGuest", "body": "Hi Seb, \nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?", "cc": [], "bcc": [], "postId": "644c90c2e5b07e0034c31219", "to": [], "feedback": {}, "threadLength": 1, "_meta": "{\"reservations\":[{\"_id\":\"640fd9793ce80b003546d5a5\",\"checkIn\":\"2023-08-03T19:00:00.000Z\",\"checkOut\":\"2023-08-05T15:00:00.000Z\"}],\"guestName\":\"Mehdi\"}", "_integration": "{\"platform\":\"airbnb2\",\"_id\":\"640fc8a4688b4a002d46b7b9\",\"airbnb2\":{\"id\":\"1428096168\",\"guestId\":302389}}", "_pendingTasks": "[]", "_thread": "[{\"_id\":\"644c90c2e5b07e0034c31219\",\"module\":\"airbnb2\",\"type\":\"fromGuest\",\"body\":\"Hi Seb, \\nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?\",\"createdAt\":\"2023-04-29T03:36:28.000Z\",\"cc\":[],\"bcc\":[],\"postId\":\"644c90c2e5b07e0034c31219\",\"to\":[],\"feedback\":{}}]", "_cc": "[]", "_bcc": "[]", "_to": "[]", "_feedback": "{}"}, "message": {"module": "airbnb2", "type": "fromGuest", "body": "Hi Seb, \nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?", "createdAt": "2023-04-29T03:36:28.000Z", "cc": [], "bcc": [], "postId": "644c90c2e5b07e0034c31219", "to": [], "feedback": {}}, "event": "reservation.messageReceived"} \ No newline at end of file diff --git a/jobs/8c9f94aa9d724e3782ffe292f7c12bf0.json b/jobs/8c9f94aa9d724e3782ffe292f7c12bf0.json new file mode 100644 index 00000000..494fc48a --- /dev/null +++ b/jobs/8c9f94aa9d724e3782ffe292f7c12bf0.json @@ -0,0 +1 @@ +{"reservationId": "640fd9793ce80b003546d5a5", "conversation": {"meta": {"reservations": [{"_id": "640fd9793ce80b003546d5a5", "checkIn": "2023-08-03T19:00:00.000Z", "checkOut": "2023-08-05T15:00:00.000Z"}], "guestName": "Mehdi"}, "guestId": "63fa63f22c3ab9002b42668c", "language": "en", "lastUpdatedFromGuest": "2023-03-17T00:00:42.000Z", "isRead": true, "snoozedUntil": null, "status": "ARCHIVED", "integration": {"platform": "airbnb2", "_id": "640fc8a4688b4a002d46b7b9", "airbnb2": {"id": "1428096168", "guestId": 302389}}, "accountId": "63f9222c980d05002c4e554d", "createdAt": "2023-04-29T03:36:28.000Z", "lastUpdatedAt": "2023-03-24T00:17:41.395Z", "lastModifiedAt": "2023-04-27T15:18:39.093Z", "conversationWith": "Guest", "priority": 10, "pendingTasks": [], "firstReceptionist": "Mohamed El Hadi Msaid", "_id": "640fd9794f14fc002f7d238d", "thread": [{"_id": "644c90c2e5b07e0034c31219", "module": "airbnb2", "type": "fromGuest", "body": "Hi Seb, \nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?", "createdAt": "2023-04-29T03:36:28.000Z", "cc": [], "bcc": [], "postId": "644c90c2e5b07e0034c31219", "to": [], "feedback": {}}], "module": "airbnb2", "type": "fromGuest", "body": "Hi Seb, \nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?", "cc": [], "bcc": [], "postId": "644c90c2e5b07e0034c31219", "to": [], "feedback": {}, "threadLength": 1, "_meta": "{\"reservations\":[{\"_id\":\"640fd9793ce80b003546d5a5\",\"checkIn\":\"2023-08-03T19:00:00.000Z\",\"checkOut\":\"2023-08-05T15:00:00.000Z\"}],\"guestName\":\"Mehdi\"}", "_integration": "{\"platform\":\"airbnb2\",\"_id\":\"640fc8a4688b4a002d46b7b9\",\"airbnb2\":{\"id\":\"1428096168\",\"guestId\":302389}}", "_pendingTasks": "[]", "_thread": "[{\"_id\":\"644c90c2e5b07e0034c31219\",\"module\":\"airbnb2\",\"type\":\"fromGuest\",\"body\":\"Hi Seb, \\nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?\",\"createdAt\":\"2023-04-29T03:36:28.000Z\",\"cc\":[],\"bcc\":[],\"postId\":\"644c90c2e5b07e0034c31219\",\"to\":[],\"feedback\":{}}]", "_cc": "[]", "_bcc": "[]", "_to": "[]", "_feedback": "{}"}, "message": {"module": "airbnb2", "type": "fromGuest", "body": "Hi Seb, \nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?", "createdAt": "2023-04-29T03:36:28.000Z", "cc": [], "bcc": [], "postId": "644c90c2e5b07e0034c31219", "to": [], "feedback": {}}, "event": "reservation.messageReceived"} \ No newline at end of file diff --git a/render.yaml b/render.yaml index 05fd6ef4..01e35c98 100644 --- a/render.yaml +++ b/render.yaml @@ -1,6 +1,6 @@ services: - type: worker - name: queue + name: zenn-worker region: ohio env: python buildCommand: "pip install -r requirements.txt" @@ -9,38 +9,38 @@ services: envVars: - key: CELERY_BROKER_URL fromService: - name: celery-redis + name: zenn-queue-redis type: redis property: connectionString - type: web - name: app + name: zenn-queue region: ohio env: python buildCommand: "pip install -r requirements.txt" startCommand: "gunicorn app:app" - autoDeploy: false + autoDeploy: true envVars: - key: CELERY_BROKER_URL fromService: - name: celery-redis + name: zenn-queue-redis type: redis property: connectionString - type: web - name: flower + name: zenn-queue-ui region: ohio plan: free env: python buildCommand: "pip install -r requirements.txt" startCommand: "celery flower --app tasks --loglevel info" - autoDeploy: false + autoDeploy: true envVars: - key: CELERY_BROKER_URL fromService: type: redis - name: celery-redis + name: zenn-queue-redis property: connectionString - type: redis - name: celery-redis + name: zenn-queue-redis region: ohio plan: starter # we choose a plan with persistence to ensure tasks are not lost upon restart maxmemoryPolicy: noeviction # recommended policy for queues diff --git a/requirements.txt b/requirements.txt index 7fa9d29e..f4870981 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ click-plugins==1.1.1 click-repl==0.2.0 Deprecated==1.2.13 Flask==2.1.1 -flower==1.0.0 +flower==2.0.1 gunicorn==20.1.0 humanize==4.0.0 importlib-metadata==4.11.3 @@ -21,11 +21,12 @@ packaging==21.3 prometheus-client==0.14.1 prompt-toolkit==3.0.29 pyparsing==3.0.8 +python-dotenv==0.21.1 pytz==2022.1 redis==4.2.2 six==1.16.0 tornado==6.1 -typing-extensions==4.1.1 +typing_extensions==4.1.1 vine==5.0.0 wcwidth==0.2.5 Werkzeug==2.1.1 diff --git a/tasks.py b/tasks.py index ed8016a9..9febe57b 100644 --- a/tasks.py +++ b/tasks.py @@ -1,12 +1,37 @@ +import json +import logging import os +import uuid from celery import Celery from celery.utils.log import get_task_logger +from dotenv import load_dotenv + +load_dotenv() -app = Celery('tasks', broker=os.getenv("CELERY_BROKER_URL")) logger = get_task_logger(__name__) +# Configure the logs +logging.basicConfig(filename='tasks.log', level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +#CELERY_BROKER_URL=os.environ.get("CELERY_BROKER_URL") +CELERY_BROKER_URL=os.getenv("CELERY_BROKER_URL") + +try: + celery = Celery('tasks', broker=CELERY_BROKER_URL) + logging.info(f'Connecting to Redis CELERY_BROKER_URL: {CELERY_BROKER_URL}') +except Exception as e: + logging.error(f'Error connecting to Redis CELERY_BROKER_URL: {CELERY_BROKER_URL}. Exception: {e}') + raise + + + + + + +@celery.task(autoretry_for=(Exception,), retry_kwargs={'max_retries': 2}) +def add_task(json_data): + #file_name should be unique, use uuid to generate unique file name + file_name='jobs/'+uuid.uuid4().hex+'.json' + with open(file_name, 'w') as file: + json.dump(json_data, file) -@app.task -def add(x, y): - logger.info(f'Adding {x} + {y}') - return x + y + logging.info(f'Job {file_name} added to queue')