Implement session affinity for fly.io (#44)

* WIP: Setup fly.io session affinity

* Fix extracting cookie value

* Remove redundant deploy statement

{{cookiecutter.project_slug}}/.github/workflows/deploy_to_fly_io.yml

@@ -7,21 +7,16 @@ on:
       - main
   workflow_dispatch:
 
+env:
+  FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
 jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: superfly/flyctl-actions/setup-flyctl@master
 
-      - name: Check if the app name is registered in fly.io
-        run: ./scripts/deploy_to_fly_io.sh
-
-      - name: Deploy to Fly
-        run: |
-            flyctl deploy --remote-only --config fly.toml --yes
-            flyctl scale count 1 --config fly.toml --yes
-            flyctl secrets set OPENAI_API_KEY=$OPENAI_API_KEY
-        env:
-          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}{% endraw %}
+      - name: Check if the app name is registered in fly.io and deploy
+        run: ./scripts/deploy_to_fly_io.sh{% endraw %}

{{cookiecutter.project_slug}}/docker/content/nginx.conf.template

@@ -5,6 +5,20 @@ upstream mesop_backend {
 
 }
 
+# Extract fly-machine-id cookie value
+map $http_cookie $fly_machine_id {
+    "~*fly-machine-id=([^;]+)" $1;
+    default "";
+}
+
+# Determine action based on cookie value
+map $fly_machine_id $sticky_action {
+    ""      "set_cookie";  # Empty cookie - need to set it
+    $FLY_MACHINE_ID   "proceed";    # Cookie matches current instance
+    default "replay";      # Cookie exists but doesn't match - need to replay
+}
+
+# Main server block
 server {
     listen $MESOP_PORT;
     server_name localhost;
@@ -15,6 +29,17 @@ server {
     add_header X-XSS-Protection "1; mode=block";
 
     location / {
+        # Handle cookie setting
+        if ($sticky_action = "set_cookie") {
+            add_header Set-Cookie "fly-machine-id=$FLY_MACHINE_ID; Max-Age=518400; Path=/";
+        }
+
+        # Handle replay
+        if ($sticky_action = "replay") {
+            add_header Fly-Replay "instance=$fly_machine_id";
+            return 307;
+        }
+
         proxy_pass http://mesop_backend;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;

{{cookiecutter.project_slug}}/docker/content/run_fastagency.sh

@@ -13,13 +13,17 @@ export MESOP_PORT=${MESOP_PORT:-8888}
 WORKERS=${WORKERS:-1}
 echo "Number of workers: $WORKERS"
 
+# Check FLY_MACHINE_ID is set, if not set, set it to dummy value
+export FLY_MACHINE_ID=${FLY_MACHINE_ID:-dummy_fly_machine_id_value}
+echo "Fly machine ID: $FLY_MACHINE_ID"
+
 # Generate nginx config
 for ((i=1; i<$WORKERS+1; i++))
 do
 	PORT=$((MESOP_PORT + i))
     sed -i "5i\    server 127.0.0.1:$PORT;" nginx.conf.template
 done
-envsubst '${MESOP_PORT}' < nginx.conf.template >/etc/nginx/conf.d/default.conf
+envsubst '${MESOP_PORT},${FLY_MACHINE_ID}' < nginx.conf.template >/etc/nginx/conf.d/default.conf
 echo "Nginx config:"
 cat /etc/nginx/conf.d/default.conf
 

{{cookiecutter.project_slug}}/scripts/deploy_to_fly_io.sh

@@ -18,7 +18,6 @@ fi
 
 echo -e "\033[0;32mDeploying to fly.io\033[0m"
 flyctl deploy --config fly.toml --yes
-flyctl scale count 1 --config fly.toml --yes
 
 echo -e "\033[0;32mSetting secrets\033[0m"
 flyctl secrets set OPENAI_API_KEY=$OPENAI_API_KEY