From 16f594b683842d21e66b05b82a64ace8403b68e8 Mon Sep 17 00:00:00 2001 From: Mehdi Mehni Date: Sat, 29 Apr 2023 12:18:58 -0400 Subject: [PATCH 1/6] first deploy --- render.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/render.yaml b/render.yaml index 05fd6ef4..0b75d0be 100644 --- a/render.yaml +++ b/render.yaml @@ -1,6 +1,6 @@ services: - type: worker - name: queue + name: zenn-worker region: ohio env: python buildCommand: "pip install -r requirements.txt" @@ -9,11 +9,11 @@ services: envVars: - key: CELERY_BROKER_URL fromService: - name: celery-redis + name: zenn-queue-redis type: redis property: connectionString - type: web - name: app + name: zenn-queue region: ohio env: python buildCommand: "pip install -r requirements.txt" @@ -26,7 +26,7 @@ services: type: redis property: connectionString - type: web - name: flower + name: zenn-queue-ui region: ohio plan: free env: python @@ -40,7 +40,7 @@ services: name: celery-redis property: connectionString - type: redis - name: celery-redis + name: zenn-queue-redis region: ohio plan: starter # we choose a plan with persistence to ensure tasks are not lost upon restart maxmemoryPolicy: noeviction # recommended policy for queues From 2a1b4c6b9612ed5dae53b71128b498cd787a810d Mon Sep 17 00:00:00 2001 From: Mehdi Mehni Date: Sat, 29 Apr 2023 12:21:14 -0400 Subject: [PATCH 2/6] zenn-queue-redis --- render.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/render.yaml b/render.yaml index 0b75d0be..d00e21c9 100644 --- a/render.yaml +++ b/render.yaml @@ -22,7 +22,7 @@ services: envVars: - key: CELERY_BROKER_URL fromService: - name: celery-redis + name: zenn-queue-redis type: redis property: connectionString - type: web @@ -37,7 +37,7 @@ services: - key: CELERY_BROKER_URL fromService: type: redis - name: celery-redis + name: zenn-queue-redis property: connectionString - type: redis name: zenn-queue-redis From 1d49f4dfd2904edc9f7928c6862e4caae57d9c8b Mon Sep 17 00:00:00 2001 From: Mehdi Mehni Date: Sat, 29 Apr 2023 18:48:52 -0400 Subject: [PATCH 3/6] t --- .gitignore | 4 +++ __pycache__/tasks.cpython-37.pyc | Bin 0 -> 1150 bytes jobs/07d9860141d945e2a168eeff1b7cca82.json | 1 + jobs/8c9f94aa9d724e3782ffe292f7c12bf0.json | 1 + tasks.py | 35 ++++++++++++++++++--- 5 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 .gitignore create mode 100644 __pycache__/tasks.cpython-37.pyc create mode 100644 jobs/07d9860141d945e2a168eeff1b7cca82.json create mode 100644 jobs/8c9f94aa9d724e3782ffe292f7c12bf0.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..249672e9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +tasks.log +.vscode/* +.env +.venv/* diff --git a/__pycache__/tasks.cpython-37.pyc b/__pycache__/tasks.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d883748a2c1d6d23dea91182ceb44d22c53611a2 GIT binary patch literal 1150 zcma)5O>f&q5ZzsVh}1_UD~=7cDL@nm5*}<_awv?V2;!VI z$;1+zOWmu{AJ7JnkNqe84SVe=z4y{nXDKu&&`X!tw=+Al*x7lHk9)l~qw(i2U&Efw z*zd;qXPC$%n&lq^z<_5g+N?a{jV~h6tX5>vDl$8FB8QtCD|7Q!)M{)y^Kw7(IYa-7 z5w@^>#iI`LMHd{x9=OoDwkRUjx4^sMQGXer13vbleJvsz`Fa2y?ElQ(iW|<^cdR(D z8E&1m)o+|dL(W!rDd#qHO%C*MNSU_D{DE?T1CNF;hsW#%E5tEFd$EmMGY+?)pK=&n z3)q6;kBn-%1N>VS-hM~#Wk(^`?jdHVmSpwkSjAORp2eA*&r#7$J$X_e*}mcd?d??HyIFe{SxevW0C%qbdL+MA^rnwWN)H0{o$%9BcO9gd%l zCokg1lV@LzC-L*iQ|&Z&n6gfkJVUj*e<+Ir>8fHLR5F+#q-F4*a|gj{Z>*G5!SsKo z_x6MF#T3u0R2Bz;?&QfutguokmhS|$CKS4t)Ri=Q%a{t*-HmznI#Kg-%1!@hSTuc_ zhcwG41i&k{<~M91c1bR~>a>?!zS#QRqnb4_9Z7` zowt@D6&R7^9<7{(JSj&jXMa)3Li5)v`-waWR{bCWKrkKozQ!6uPuq2!f_4l()Yd6p zXj`6Rq3!0R4RwC5TZUNbq_)#yCPORoO)!R}O0;iSiW{12WB$t22lSl3J{dhPQI(?{ zPl5P8Ez;5Fa$4tDRJ6IIh69cwEQ-C2kVYGooG-OUH8y(s$n?e|VAfD{`HoO`$qska zHZ5w0Knq!_fic`^BS&R8hz1*Hzpm1(5uP&bj@*qP75&UidvQ~A9BX$YTirQH%5+M) znWgm7FOR->rd^UYNm!X8)UG+^zh_@_5+)k)Xxr4@6xY1{U+`gb)u*|HI>XNl`Q^t1 Z9^ZUjPM>gvCx)Ue1bN$hz<2m|>u+K`Jn#Sj literal 0 HcmV?d00001 diff --git a/jobs/07d9860141d945e2a168eeff1b7cca82.json b/jobs/07d9860141d945e2a168eeff1b7cca82.json new file mode 100644 index 00000000..494fc48a --- /dev/null +++ b/jobs/07d9860141d945e2a168eeff1b7cca82.json @@ -0,0 +1 @@ +{"reservationId": "640fd9793ce80b003546d5a5", "conversation": {"meta": {"reservations": [{"_id": "640fd9793ce80b003546d5a5", "checkIn": "2023-08-03T19:00:00.000Z", "checkOut": "2023-08-05T15:00:00.000Z"}], "guestName": "Mehdi"}, "guestId": "63fa63f22c3ab9002b42668c", "language": "en", "lastUpdatedFromGuest": "2023-03-17T00:00:42.000Z", "isRead": true, "snoozedUntil": null, "status": "ARCHIVED", "integration": {"platform": "airbnb2", "_id": "640fc8a4688b4a002d46b7b9", "airbnb2": {"id": "1428096168", "guestId": 302389}}, "accountId": "63f9222c980d05002c4e554d", "createdAt": "2023-04-29T03:36:28.000Z", "lastUpdatedAt": "2023-03-24T00:17:41.395Z", "lastModifiedAt": "2023-04-27T15:18:39.093Z", "conversationWith": "Guest", "priority": 10, "pendingTasks": [], "firstReceptionist": "Mohamed El Hadi Msaid", "_id": "640fd9794f14fc002f7d238d", "thread": [{"_id": "644c90c2e5b07e0034c31219", "module": "airbnb2", "type": "fromGuest", "body": "Hi Seb, \nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?", "createdAt": "2023-04-29T03:36:28.000Z", "cc": [], "bcc": [], "postId": "644c90c2e5b07e0034c31219", "to": [], "feedback": {}}], "module": "airbnb2", "type": "fromGuest", "body": "Hi Seb, \nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?", "cc": [], "bcc": [], "postId": "644c90c2e5b07e0034c31219", "to": [], "feedback": {}, "threadLength": 1, "_meta": "{\"reservations\":[{\"_id\":\"640fd9793ce80b003546d5a5\",\"checkIn\":\"2023-08-03T19:00:00.000Z\",\"checkOut\":\"2023-08-05T15:00:00.000Z\"}],\"guestName\":\"Mehdi\"}", "_integration": "{\"platform\":\"airbnb2\",\"_id\":\"640fc8a4688b4a002d46b7b9\",\"airbnb2\":{\"id\":\"1428096168\",\"guestId\":302389}}", "_pendingTasks": "[]", "_thread": "[{\"_id\":\"644c90c2e5b07e0034c31219\",\"module\":\"airbnb2\",\"type\":\"fromGuest\",\"body\":\"Hi Seb, \\nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?\",\"createdAt\":\"2023-04-29T03:36:28.000Z\",\"cc\":[],\"bcc\":[],\"postId\":\"644c90c2e5b07e0034c31219\",\"to\":[],\"feedback\":{}}]", "_cc": "[]", "_bcc": "[]", "_to": "[]", "_feedback": "{}"}, "message": {"module": "airbnb2", "type": "fromGuest", "body": "Hi Seb, \nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?", "createdAt": "2023-04-29T03:36:28.000Z", "cc": [], "bcc": [], "postId": "644c90c2e5b07e0034c31219", "to": [], "feedback": {}}, "event": "reservation.messageReceived"} \ No newline at end of file diff --git a/jobs/8c9f94aa9d724e3782ffe292f7c12bf0.json b/jobs/8c9f94aa9d724e3782ffe292f7c12bf0.json new file mode 100644 index 00000000..494fc48a --- /dev/null +++ b/jobs/8c9f94aa9d724e3782ffe292f7c12bf0.json @@ -0,0 +1 @@ +{"reservationId": "640fd9793ce80b003546d5a5", "conversation": {"meta": {"reservations": [{"_id": "640fd9793ce80b003546d5a5", "checkIn": "2023-08-03T19:00:00.000Z", "checkOut": "2023-08-05T15:00:00.000Z"}], "guestName": "Mehdi"}, "guestId": "63fa63f22c3ab9002b42668c", "language": "en", "lastUpdatedFromGuest": "2023-03-17T00:00:42.000Z", "isRead": true, "snoozedUntil": null, "status": "ARCHIVED", "integration": {"platform": "airbnb2", "_id": "640fc8a4688b4a002d46b7b9", "airbnb2": {"id": "1428096168", "guestId": 302389}}, "accountId": "63f9222c980d05002c4e554d", "createdAt": "2023-04-29T03:36:28.000Z", "lastUpdatedAt": "2023-03-24T00:17:41.395Z", "lastModifiedAt": "2023-04-27T15:18:39.093Z", "conversationWith": "Guest", "priority": 10, "pendingTasks": [], "firstReceptionist": "Mohamed El Hadi Msaid", "_id": "640fd9794f14fc002f7d238d", "thread": [{"_id": "644c90c2e5b07e0034c31219", "module": "airbnb2", "type": "fromGuest", "body": "Hi Seb, \nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?", "createdAt": "2023-04-29T03:36:28.000Z", "cc": [], "bcc": [], "postId": "644c90c2e5b07e0034c31219", "to": [], "feedback": {}}], "module": "airbnb2", "type": "fromGuest", "body": "Hi Seb, \nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?", "cc": [], "bcc": [], "postId": "644c90c2e5b07e0034c31219", "to": [], "feedback": {}, "threadLength": 1, "_meta": "{\"reservations\":[{\"_id\":\"640fd9793ce80b003546d5a5\",\"checkIn\":\"2023-08-03T19:00:00.000Z\",\"checkOut\":\"2023-08-05T15:00:00.000Z\"}],\"guestName\":\"Mehdi\"}", "_integration": "{\"platform\":\"airbnb2\",\"_id\":\"640fc8a4688b4a002d46b7b9\",\"airbnb2\":{\"id\":\"1428096168\",\"guestId\":302389}}", "_pendingTasks": "[]", "_thread": "[{\"_id\":\"644c90c2e5b07e0034c31219\",\"module\":\"airbnb2\",\"type\":\"fromGuest\",\"body\":\"Hi Seb, \\nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?\",\"createdAt\":\"2023-04-29T03:36:28.000Z\",\"cc\":[],\"bcc\":[],\"postId\":\"644c90c2e5b07e0034c31219\",\"to\":[],\"feedback\":{}}]", "_cc": "[]", "_bcc": "[]", "_to": "[]", "_feedback": "{}"}, "message": {"module": "airbnb2", "type": "fromGuest", "body": "Hi Seb, \nAre there any safety features such as a carbon monoxide alarm or a smoke alarm?", "createdAt": "2023-04-29T03:36:28.000Z", "cc": [], "bcc": [], "postId": "644c90c2e5b07e0034c31219", "to": [], "feedback": {}}, "event": "reservation.messageReceived"} \ No newline at end of file diff --git a/tasks.py b/tasks.py index ed8016a9..9febe57b 100644 --- a/tasks.py +++ b/tasks.py @@ -1,12 +1,37 @@ +import json +import logging import os +import uuid from celery import Celery from celery.utils.log import get_task_logger +from dotenv import load_dotenv + +load_dotenv() -app = Celery('tasks', broker=os.getenv("CELERY_BROKER_URL")) logger = get_task_logger(__name__) +# Configure the logs +logging.basicConfig(filename='tasks.log', level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +#CELERY_BROKER_URL=os.environ.get("CELERY_BROKER_URL") +CELERY_BROKER_URL=os.getenv("CELERY_BROKER_URL") + +try: + celery = Celery('tasks', broker=CELERY_BROKER_URL) + logging.info(f'Connecting to Redis CELERY_BROKER_URL: {CELERY_BROKER_URL}') +except Exception as e: + logging.error(f'Error connecting to Redis CELERY_BROKER_URL: {CELERY_BROKER_URL}. Exception: {e}') + raise + + + + + + +@celery.task(autoretry_for=(Exception,), retry_kwargs={'max_retries': 2}) +def add_task(json_data): + #file_name should be unique, use uuid to generate unique file name + file_name='jobs/'+uuid.uuid4().hex+'.json' + with open(file_name, 'w') as file: + json.dump(json_data, file) -@app.task -def add(x, y): - logger.info(f'Adding {x} + {y}') - return x + y + logging.info(f'Job {file_name} added to queue') From 19f98a1788a6156ea04642d6b304face6830c02b Mon Sep 17 00:00:00 2001 From: Mehdi Mehni Date: Sat, 29 Apr 2023 18:53:10 -0400 Subject: [PATCH 4/6] python-dotenv --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7fa9d29e..c6401c0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,11 +21,12 @@ packaging==21.3 prometheus-client==0.14.1 prompt-toolkit==3.0.29 pyparsing==3.0.8 +python-dotenv==0.21.1 pytz==2022.1 redis==4.2.2 six==1.16.0 tornado==6.1 -typing-extensions==4.1.1 +typing_extensions==4.1.1 vine==5.0.0 wcwidth==0.2.5 Werkzeug==2.1.1 From 5e49f6792731f6ea793f8341fffdbadc02bb5e0a Mon Sep 17 00:00:00 2001 From: Mehdi Mehni Date: Thu, 12 Oct 2023 11:04:43 -0400 Subject: [PATCH 5/6] Update flower --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c6401c0e..f4870981 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ click-plugins==1.1.1 click-repl==0.2.0 Deprecated==1.2.13 Flask==2.1.1 -flower==1.0.0 +flower==2.0.1 gunicorn==20.1.0 humanize==4.0.0 importlib-metadata==4.11.3 From 953b70c3e3632213d1e0e03ff8ec547efafb5f9e Mon Sep 17 00:00:00 2001 From: Mehdi Mehni Date: Thu, 12 Oct 2023 11:17:20 -0400 Subject: [PATCH 6/6] Update render.yaml --- render.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/render.yaml b/render.yaml index d00e21c9..01e35c98 100644 --- a/render.yaml +++ b/render.yaml @@ -18,7 +18,7 @@ services: env: python buildCommand: "pip install -r requirements.txt" startCommand: "gunicorn app:app" - autoDeploy: false + autoDeploy: true envVars: - key: CELERY_BROKER_URL fromService: @@ -32,7 +32,7 @@ services: env: python buildCommand: "pip install -r requirements.txt" startCommand: "celery flower --app tasks --loglevel info" - autoDeploy: false + autoDeploy: true envVars: - key: CELERY_BROKER_URL fromService: