Feature/multi servers 2

README.md

@@ -10,11 +10,29 @@ World's smallest book store with a two-tier web design
 - Milestone 1 - https://github.com/CS677-Labs/Lab-1-The_Bazaar/tree/milestone1
 
 
-## Run testcases for Milestone 1
+## Run testcases for Single server, single client usecase
 ```
 cd Lab-2-Pygmy-The-book-store
-chmod 777 test/Milestone_1_Testcases.sh
-bash test/Milestone_1_Testcases.sh
+chmod 777 test/SingleServerSingleClient.sh
+bash test/SingleServerSingleClient.sh
+```
+
+## Run testcases for Single server, multi client usecase
+```
+cd Lab-2-Pygmy-The-book-store
+chmod 777 test/SingleServerMultiClients.sh
+bash test/SingleServerMultiClients.sh
+```
+
+## Run testcases for Multi server, multi client usecase
+Create a file "machines.txt" with 3 lines. 
+First line has the IP of the server where catalog server is to be launched.
+Second line has the IP of the server where order server is to be launched.
+Third line has the IP of the server where frontend server is to be launched.
+```
+cd Lab-2-Pygmy-The-book-store
+chmod 777 test/MultiServerMultiClients.sh
+bash test/MultiServerMultiClients.sh machines.txt
 ```
 
 ## Usage

machines.txt.local

@@ -0,0 +1,3 @@
+localhost
+localhost
+localhost

src/catalog_server/requirements.txt → requirements.txt

@@ -10,3 +10,4 @@ marshmallow==3.11.0
 six==1.15.0
 SQLAlchemy==1.4.3
 Werkzeug==1.0.1
+requests==2.22.0

run.sh

@@ -10,28 +10,26 @@ function finish {
   done
 
 #  # For remote cleanup
-  while IFS= read -r line
-  do
-    newline=(${line//=/ })
-    id=${newline[0]}
-    network=(${newline[1]//,/ })
-    url=(${network[0]//:/ })
-    ip="${url[0]}:${url[1]}"
-    fullurl=$ip
-    ip=$(echo "$fullurl" |sed 's/https\?:\/\///')
-    ssh -n ec2-user@"$ip" "kill ${pids[id]}" || echo "Failed to kill process $i."
-  done < "network-config.properties"
-  rm -rf build/* >/dev/null 2>&1
+  for i in ${!servers[@]}; do	    
+    ip=${machines[$i]}
+    if [[ "$ip" != *"localhost" ]]
+    then
+    	echo "Attempting to cleanup ${servers[$i]} with PID ${pids[$i]} on $ip"
+    	ssh -n ec2-user@"$ip" "kill -9 ${pids[$i]}" || echo "Failed to kill process $i."
+    fi
+  done
+  return
 }
 trap finish EXIT
-trap finish RETURN
-
+trap finish INT
 
-servers=("catalog_server", "order_server", "frontend_server")
-ports=("5000", "5001", "5002")
+servers=("catalog_server" "order_server" "frontend_server")
+ports=("5000" "5001" "5002")
 machines=()
 local_pids=()
 pids=()
+configFile=$1
+echo $configFile
 #
  # Read N - Number of servers. Keep assigning nodes to them - Catalog, Order, and Frontend and run the app accordingly. 
  # Print the url of frontend server. 
@@ -40,20 +38,21 @@ echo "Setting up the servers on machines..."
 while IFS= read -r line
 do
   machines+=($line)
-done < machines.txt
+done < $configFile
 
 for i in ${!servers[@]}; do
   role=${servers[$i]}
   ip=${machines[$i]}
   port=${ports[$i]}
-  if [[ "$ip" == *"http://localhost" ]] || [[ "$ip" == *"http://127.0.0.1" ]]
+  if [[ "$ip" == *"localhost" ]] || [[ "$ip" == *"127.0.0.1" ]]
   then
     echo "Running $role on Localhost...."
+    cp -f $configFile "config"
     export FLASK_APP=src/$role/views.py
-    python3 -m flask run --port $port 2>/dev/null &
+    python3 -m flask run --port $port >/dev/null 2>&1 &
     pid=$!
     sleep 3
-    if ! (ps -ef | grep "python" | grep "$pid_order" | grep -v grep >/dev/null 2>&1)
+    if ! (ps -ef | grep "python" | grep "$pid" | grep -v grep >/dev/null 2>&1)
     then
 	    echo "Failed to start $role" && return 1
     fi
@@ -64,21 +63,23 @@ for i in ${!servers[@]}; do
     local_pids+=($pid)
   else
     echo "Running role $role on remote machine $ip."
-    dir[id]="temp_$id"
-    ssh -n ec2-user@"$ip" "rm -rf temp_$id && mkdir temp_$id && cd temp_id && git clone https://github.com/CS677-Labs/Lab-2-Pygmy-The-book-store.git || echo \"Repo already present\""
-    scp "machines.txt" ec2-user@"$ip":"temp_$id"
-    pid=$(ssh -n ec2-user@$ip "cd temp_$id/Lab-2-Pygmy-The-book-store/src/$role && export FLASK_APP=views.py && (python3 -m flask run --port $port 2>/dev/null & echo \$!)")
+    dir[$i]="temp_$i"
+    #Todo: Remove checkout
+    ssh -n ec2-user@"$ip" "rm -rf temp_$i && mkdir temp_$i && cd temp_$i && git clone https://github.com/CS677-Labs/Lab-2-Pygmy-The-book-store 1>/dev/null 2>&1  && cd L* && git checkout feature/multi-servers-2 1>/dev/null 2>&1 || echo \"Repo already present\""
+    scp "machines.txt" ec2-user@"$ip":"temp_$i/Lab-2-Pygmy-The-book-store/src/$role/config" >/dev/null 2>&1
+    pid=$(ssh -n ec2-user@$ip "sudo pip3 install -r temp_$i/Lab-2-Pygmy-The-book-store/requirements.txt 1>/dev/null 2>&1  && cd temp_$i/Lab-2-Pygmy-The-book-store/src/$role && export FLASK_APP=views.py && (python3 -m flask run --host 0.0.0.0 --port $port >/dev/null 2>&1 & echo \$!)")
+    echo $pid
     sleep 2
     status=0
-    ssh -n ec2-user@"$ip" "ps -ef | grep java | grep $pid | grep -v grep" || status=$?
-    pids[id]=$pid
+    ssh -n ec2-user@"$ip" "ps -ef | grep python | grep $pid | grep -v grep >/dev/null 2>&1" || status=$?
+    pids[i]=$pid
   fi
   if [[ "$status" != 0 ]]
   then
 	  echo "Failed to start the server $role in machine with ip $ip. Exiting..." && return 1
   fi
 done
-echo "Frontend server is running on ${machine[2]}.... Pass this to the CLI to use it with this server."
+echo "Frontend server is running on ${machines[2]}.... Pass this to the CLI to use it with this server."
 
 echo "Press 'q' to exit"
 count=0
@@ -90,4 +91,4 @@ then
   exit
 fi
 done
-echo "---------------------------------------------------------------"
+echo "---------------------------------------------------------------"

src/catalog_server/models.py

@@ -3,16 +3,36 @@
 
 from flask import Flask
 from flask_marshmallow import Marshmallow
-from flask_sqlalchemy import SQLAlchemy
+from flask_sqlalchemy import SQLAlchemy, get_debug_queries
+
 
 app = Flask(__name__)
 basedir = os.path.abspath(os.path.dirname(__file__))
 app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + os.path.join(basedir, 'data', 'catalog.sqlite')
+# app.config["SQLALCHEMY_ECHO"] = True
+app.config["SQLALCHEMY_RECORD_QUERIES"] = True
+
 db = SQLAlchemy(app)
 ma = Marshmallow(app)
 
 
+@app.after_request
+def after_request(response):
+    """
+    This function is executed after the API request is completed.
+    The queries executed are logged and the response is returned to the client that sent the API request.
+    """
+    for query in get_debug_queries():
+        with open(os.path.join(basedir, 'logs', 'query_log.txt'), 'a') as f:
+            f.write('Query: %s\nParameters: %s\n\n'
+                % (query.statement, query.parameters))
+    return response
+
+
 class Book(db.Model):
+    """
+    Database model to store book details for the book store application.
+    """
     id = db.Column(db.Integer, primary_key=True, autoincrement=True)
     title = db.Column(db.VARCHAR)
     topic = db.Column(db.VARCHAR)
@@ -37,7 +57,9 @@ class Meta:
 
 if not os.path.exists(os.path.join(basedir, 'data', 'catalog.sqlite')):
     db.create_all()
-    with open(os.path.join(basedir, 'data', 'book_details.json')) as f:
+    with open(os.path.join(basedir, 'data', 'book_details.json')) as f:  # book_details.json consists of the details of
+        # each book like cost, number of copies available, title of the book and topic.
+        # The database is initialised using this data.
         data = json.load(f)
     for book in data:
         new_book = Book(book['title'], book['topic'], book['count'], book['cost'])

src/catalog_server/views.py

@@ -3,7 +3,7 @@
 from flask import request, jsonify, Response
 from sqlalchemy import exc
 
-from src.catalog_server.models import Book, BookSchema, db, app
+from models import Book, BookSchema, db, app
 
 book_schema = BookSchema()
 books_schema = BookSchema(many=True)
@@ -40,7 +40,8 @@ def book_detail(id):
 # Endpoint to update details of a book
 @app.route("/books/<id>", methods=["PATCH"])
 def book_update(id):
-    book = Book.query.get(id)
+    # Handling concurrent requests
+    book = db.session.query(Book).filter(Book.id==id).with_for_update().one()
     current_book_count = book.count
     book.topic = request.json.get('topic') or book.topic
     book.title = request.json.get('title') or book.title

src/cli/book_requests.py

@@ -1,10 +1,7 @@
 import requests
 from typing import Dict, List
 
-FRONTEND_URL= ""
-
-
-def get_book(item_number: int) -> Dict:
+def get_book(item_number: int, FRONTEND_URL: str) -> Dict:
     """
     Fetches the book corresponding to item number item_number from the front end server.
     """
@@ -18,7 +15,7 @@ def get_book(item_number: int) -> Dict:
         raise Exception(str(response.text))
     return response.json()
 
-def get_books_by_topic(topic: str) -> List[Dict]:
+def get_books_by_topic(topic: str, FRONTEND_URL: str) -> List[Dict]:
     payload = {"topic": topic}
     try:
         response = requests.get(f"{FRONTEND_URL}/books", params=payload)
@@ -30,14 +27,13 @@ def get_books_by_topic(topic: str) -> List[Dict]:
 
     return response.json()
 
-def buy_book(item_number: int) -> Dict:
+def buy_book(item_number: int, FRONTEND_URL: str) -> Dict:
     try:
         response = requests.post(f"{FRONTEND_URL}/books/{item_number}")
     except requests.exceptions.RequestException as e:
         raise Exception(f"Frontend server seems to be down. Failed to buy the book with item number {item_number}.")
-    # Todo: Add comments for execption handling.
+    # Todo: Add comments for exception handling.
     if response.status_code != 200:
         raise Exception(str(response.text))
     return response.json()
 
-

src/cli/main.py

@@ -1,29 +1,32 @@
 import click
-from book_requests import get_book, get_books_by_topic, buy_book, FRONTEND_URL
+from book_requests import get_book, get_books_by_topic, buy_book
 
 @click.group()
 @click.argument("frontend_server", default="http://localhost:5002")
-def bookstore(frontend_server: str):
-    FRONTEND_URL=frontend_server
-    click.echo("Welcome to the World's Smallest Book Store!")
+@click.pass_context
+def bookstore(ctx, frontend_server: str):
+    ctx.obj=f"http://{frontend_server}:5002"
+    click.echo(f"Welcome to the World's Smallest Book Store!")
 
 @click.command()
 @click.argument("item_number", required=True)
-def lookup(item_number: int):
-    click.echo(f"Looking up item number {item_number} ...")
+@click.pass_context
+def lookup(ctx, item_number: int):
+    click.echo(f"Looking up item number {item_number}...")
     try:
-        book = get_book(item_number)
+        book = get_book(item_number, ctx.obj)
     except Exception as e:
         click.echo(str(e))
         return
     click.echo(book)
 
 @click.command()
 @click.option("--topic", required=True, help="The topic of the books to search for.")
-def search(topic: str):
+@click.pass_context
+def search(ctx, topic: str):
     click.echo(f"Searching for all the books related to topic {topic} ...")
     try:
-        books = get_books_by_topic(topic)
+        books = get_books_by_topic(topic, ctx.obj)
     except Exception as e:
         click.echo(str(e))
         return
@@ -32,10 +35,11 @@ def search(topic: str):
 
 @click.command()
 @click.argument("item_number", required=True)
-def buy(item_number: int):
+@click.pass_context
+def buy(ctx, item_number: int):
     click.echo(f"Looking up item number {item_number}...")
     try:
-        book = buy_book(item_number)
+        book = buy_book(item_number, ctx.obj)
     except Exception as e:
         click.echo(str(e))
         return
@@ -47,5 +51,6 @@ def buy(item_number: int):
 bookstore.add_command(buy)
 
 if __name__ == '__main__':
-    bookstore()
+    FRONTEND_URL=""
+    bookstore(obj=FRONTEND_URL)
 

src/frontend_server/views.py

@@ -4,16 +4,22 @@
 import os
 import requests
 
-logging.basicConfig(filename='frontend.log', level=logging.WARNING)
+logging.basicConfig(filename='frontend.log', level=logging.DEBUG)
 flask = flask.Flask(__name__)
 
-
 @flask.route('/books/<int:item_number>', methods=['GET'])
 def lookup(item_number: int):    
-    # Todo: Fetch the url from a config file rather than hardcoding it.
+
+    f = open("config", "r")
+    catalogServerIP = f.readline().rstrip('\r\n')
+    f.close()
+
     try:
-        r = requests.get(f'http://localhost:5000/books/{item_number}')
+        url=f'http://{catalogServerIP}:5000/books/{item_number}'
+        logging.debug(f"Trying to connect to {url}")
+        r = requests.get(url)
     except requests.exceptions.RequestException as e:
+        logging.error(f"Exception occured. {e}")
         return f"Ughh! Catalog server seems to be down. Failed to fetch the book with item number {item_number}.", 501
     if r.status_code != 200:
         error_msg = f"Failed to fetch the book with item number {item_number}."
@@ -26,13 +32,21 @@ def lookup(item_number: int):
 @flask.route('/books', methods=['GET'])
 def search():
     topic = request.args.get('topic')
+
+    f = open("config", "r")
+    catalogServerIP = f.readline().rstrip('\r\n')
+    f.close()
+
     if topic is None:
         return "The request must send the query parameter \"topic\"", 400
-    # Todo: Fetch the url from a config file rather than hardcoding it.
+
     payload = {'topic':topic}
     try:
-        r = requests.get(f'http://localhost:5000/books', params=payload)
+        url=f'http://{catalogServerIP}:5000/books'
+        logging.info(f"Trying to connect to {url}")
+        r = requests.get(url, params=payload)
     except requests.exceptions.RequestException as e:
+        logging.error(f"Exception occured. {e}")
         return f"Ughh! Catalog server seems to be down. Failed to fetch the books for the topic {topic}.", 501
     if r.status_code != 200:
         return f"Failed to fetch the books related to topic {topic}.", r.status_code
@@ -41,10 +55,18 @@ def search():
 
 @flask.route('/books/<int:item_number>', methods=['POST'])
 def buy(item_number: int):
-    # Todo: Fetch the url from a config file rather than hardcoding it.
+
+    f = open("config", "r")
+    catalogServerIP = f.readline().rstrip('\r\n')
+    orderServerIP = f.readline().rstrip('\r\n')
+    f.close()
+
     try:
-        r = requests.post(f'http://localhost:5001/books/{item_number}')
+        url=f'http://{orderServerIP}:5001/books/{item_number}'
+        logging.info(f"Trying to connect to {url}")
+        r = requests.post(url)
     except requests.exceptions.RequestException as e:
+        logging.error(f"Exception occured. {e}")
         return f"Ughh! Order server seems to be down. Failed to buy the book with item number {item_number}.", 501
 
     if r.status_code != 200:
@@ -56,4 +78,4 @@ def buy(item_number: int):
     return book
 
 if __name__ == '__main__':
-    flask.run(debug=True)
+    flask.run(debug=True)