Introduce new Model + Database implementation

park_api/app.py

@@ -1,36 +1,21 @@
 from datetime import datetime
 from os import getloadavg
-
 from flask import Flask, jsonify, abort, request
-import psycopg2
-from park_api import scraper, util, env, db
+
+from park_api import scraper, util, env
 from park_api.forecast import find_forecast
 from park_api.crossdomain import crossdomain
 
 app = Flask(__name__)
 
 
-def user_agent(request):
-    ua = request.headers.get("User-Agent")
-    return "no user-agent" if ua is None else ua
-
-
 @app.route("/")
 @crossdomain("*")
 def get_meta():
-    app.logger.info("GET / - " + user_agent(request))
-
     cities = {}
     for module in env.supported_cities().values():
         city = module.geodata.city
-        cities[city.id] = {
-                "name": city.name,
-                "coords": city.coords,
-                "source": city.source,
-                "url": city.url,
-                "active_support": city.active_support
-        }
-
+        cities[city.id] = city.as_json()
     return jsonify({
         "cities": cities,
         "api_version": env.API_VERSION,
@@ -55,8 +40,6 @@ def get_lots(city):
     if city == "favicon.ico" or city == "robots.txt":
         abort(404)
 
-    app.logger.info("GET /" + city + " - " + user_agent(request))
-
     city_module = env.supported_cities().get(city, None)
 
     if city_module is None:
@@ -66,27 +49,18 @@ def get_lots(city):
                 "' isn't supported at the current time.", 404)
 
     if env.LIVE_SCRAPE:
-        return jsonify(scraper._live(city_module))
+        lots = scraper.scrape(city_module)
+        return jsonify(lots.as_json())
 
-    try:
-      with db.cursor() as cursor:
-          sql = "SELECT timestamp_updated, timestamp_downloaded, data" \
-                  " FROM parkapi WHERE city=%s ORDER BY timestamp_downloaded DESC LIMIT 1;"
-          cursor.execute(sql, (city,))
-          data = cursor.fetchall()[0]["data"]
-    except (psycopg2.OperationalError, psycopg2.ProgrammingError) as e:
-        app.logger.error("Unable to connect to database: " + str(e))
-        abort(500)
+    lots = city_module.geodata.lots
+    lots.load()
 
-    return jsonify(data)
+    return jsonify(lots.as_json())
 
 
 @app.route("/<city>/<lot_id>/timespan")
 @crossdomain("*")
 def get_longtime_forecast(city, lot_id):
-    app.logger.info("GET /%s/%s/timespan %s" %
-                    (city, lot_id, user_agent(request)))
-
     try:
         datetime.strptime(request.args["from"], '%Y-%m-%dT%H:%M:%S')
         datetime.strptime(request.args["to"], '%Y-%m-%dT%H:%M:%S')
@@ -103,12 +77,18 @@ def get_longtime_forecast(city, lot_id):
 
 @app.route("/coffee")
 def make_coffee():
-    app.logger.info("GET /coffee - " + user_agent(request))
-
     return """
     <h1>I'm a teapot</h1>
     <p>This server is a teapot, not a coffee machine.</p><br>
     <img src="http://i.imgur.com/xVpIC9N.gif"
          alt="British porn"
          title="British porn"/>
     """, 418
+
+
+@app.before_request
+def log_request():
+    ua = request.headers.get("User-Agent")
+    if not ua:
+        ua = "no user-agent"
+    app.logger.info("%s %s - %s" % (request.method, request.path, ua))

park_api/cities/Bonn.py

@@ -1,6 +1,6 @@
 from bs4 import BeautifulSoup
-from park_api.geodata import GeoData
-from park_api.util import convert_date
+from park_api.models import GeoData, Lots
+from park_api.util import parse_date
 
 geodata = GeoData(__file__)
 
@@ -22,21 +22,13 @@ def parse_html(html):
         "Expect to find 6 lots in Bonn, got: %d" % len(free_lots)
     time = soup.find("td", {"class": "stand"}).text.strip()
 
-    lots = []
+    lots = Lots()
+    updated_at = parse_date(time, "%d.%m.%y %H:%M:%S")
     for idx, free in enumerate(free_lots):
         lot = geodata.lot(lot_map[idx])
-        lots.append({
-            "name": lot.name,
-            "coords": lot.coords,
-            "free": int(free.text),
-            "address": lot.address,
-            "total": lot.total,
-            "state": "nodata",
-            "id": lot.id,
-            "forecast": False
-        })
+        lot.free = int(free.text)
+        lot.updated_at = updated_at
+        lot.state = "nodata"
+        lots.append(lot)
 
-    return {
-        "last_updated": convert_date(time, "%d.%m.%y %H:%M:%S"),
-        "lots": lots
-    }
+    return lots

park_api/cities/Dresden.py

@@ -1,63 +1,46 @@
-import os
 from bs4 import BeautifulSoup
-from park_api.geodata import GeoData
-from park_api.util import convert_date, get_most_lots_from_known_data
+from park_api.models import GeoData, Lots
+from park_api.util import parse_date
 
 geodata = GeoData(__file__)
 
 
 def parse_html(html):
     soup = BeautifulSoup(html, "html.parser")
-    date_field = soup.find(id="P1_LAST_UPDATE").text
-    last_updated = convert_date(date_field, "%d.%m.%Y %H:%M:%S")
-    data = {
-        "lots": [],
-        "last_updated": last_updated
-    }
+    date = soup.find(id="P1_LAST_UPDATE").text
 
+    lots = Lots()
+    updated_at = parse_date(date, "%d.%m.%Y %H:%M:%S")
     for table in soup.find_all("table"):
-        if table["summary"] != "":
-            region = table["summary"]
-
-            for lot_row in table.find_all("tr"):
-                if lot_row.find("th") is not None:
-                    continue
-
-                cls = lot_row.find("div")["class"]
-                state = "nodata"
-                if "green" in cls or "yellow" in cls or "red" in cls:
-                    state = "open"
-                elif "park-closed" in cls:
-                    state = "closed"
-
-                lot_name = lot_row.find("td", {"headers": "BEZEICHNUNG"}).text
-
-                try:
-                    col = lot_row.find("td", {"headers": "FREI"})
-                    free = int(col.text)
-                except ValueError:
-                    free = 0
-
-                try:
-                    col = lot_row.find("td", {"headers": "KAPAZITAET"})
-                    total = int(col.text)
-                except ValueError:
-                    total = get_most_lots_from_known_data("Dresden", lot_name)
-
-                lot = geodata.lot(lot_name)
-                forecast = os.path.isfile("forecast_data/" + lot.id + ".csv")
-
-                data["lots"].append({
-                    "coords": lot.coords,
-                    "name": lot_name,
-                    "total": total,
-                    "free": free,
-                    "state": state,
-                    "id": lot.id,
-                    "lot_type": lot.type,
-                    "address": lot.address,
-                    "forecast": forecast,
-                    "region": region
-                })
-
-    return data
+        if table["summary"] == "":
+            continue
+        region = table["summary"]
+
+        for lot_row in table.find_all("tr"):
+            if lot_row.find("th") is not None:
+                continue
+
+            name = lot_row.find("td", {"headers": "BEZEICHNUNG"}).text
+            lot = geodata.lot(name)
+
+            col = lot_row.find("td", {"headers": "FREI"})
+            if col.text.strip() == "":
+                lot.free = 0
+            else:
+                lot.free = int(col.text)
+
+            cls = lot_row.find("div")["class"]
+            if "green" in cls or "yellow" in cls or "red" in cls:
+                lot.state = "open"
+            elif "park-closed" in cls:
+                lot.state = "closed"
+
+            col = lot_row.find("td", {"headers": "KAPAZITAET"})
+            try:
+                lot.total = int(col.text)
+            except ValueError:
+                pass
+            lot.region = region
+            lot.updated_at = updated_at
+            lots.append(lot)
+    return lots

park_api/cities/Ingolstadt.py

@@ -1,6 +1,6 @@
 from bs4 import BeautifulSoup
-from park_api.geodata import GeoData
-from park_api.util import convert_date
+from park_api.models import GeoData, Lots
+from park_api.util import parse_date
 
 # Additional information for single lots:
 # http://www2.ingolstadt.de/Wirtschaft/Parken/Parkeinrichtungen_der_IFG/
@@ -10,34 +10,20 @@
 def parse_html(html):
     soup = BeautifulSoup(html, "html.parser")
 
-    data = {
-        "last_updated": convert_date(soup.p.string, "(%d.%m.%Y, %H.%M Uhr)"),
-        "lots": []
-    }
+    updated_at = parse_date(soup.p.string, "(%d.%m.%Y, %H.%M Uhr)")
+    lots = Lots()
 
-    # get all lots
-    raw_lots = soup.find_all("tr")
+    for raw_lot in soup.find_all("tr"):
+        tds = raw_lot.find_all("td")
 
-    for raw_lot in raw_lots:
-        elements = raw_lot.find_all("td")
-
-        state = "open"
         if "class" in raw_lot.attrs and raw_lot["class"][0] == "strike":
             state = "closed"
-
-        lot_name = elements[0].text
-
-        lot = geodata.lot(lot_name)
-        data["lots"].append({
-            "name": lot.name,
-            "free": int(elements[1].text),
-            "total": lot.total,
-            "lot_type": lot.type,
-            "address": lot.address,
-            "coords": lot.coords,
-            "state": state,
-            "id": lot.id,
-            "forecast": False
-        })
-
-    return data
+        else:
+            state = "open"
+
+        lot = geodata.lot(tds[0].text)
+        lot.free = int(tds[1].text)
+        lot.state = state
+        lot.updated_at = updated_at
+        lots.append(lot)
+    return lots

park_api/cities/Konstanz.py

@@ -1,6 +1,6 @@
 from bs4 import BeautifulSoup
-from park_api.util import convert_date
-from park_api.geodata import GeoData
+from park_api.util import parse_date
+from park_api.models import GeoData, Lots
 
 geodata = GeoData(__file__)
 
@@ -11,48 +11,25 @@ def parse_html(html):
     # last update time (UTC)
     try:
         date_col = soup.select('p > strong')[-1].text
-        update_time = convert_date(date_col, "Stand: %d.%m.%Y - %H:%M:%S")
+        updated_at = parse_date(date_col, "Stand: %d.%m.%Y - %H:%M:%S")
     except ValueError:
         date_col = soup.select('p > strong')[-2].text
-        update_time = convert_date(date_col, "Stand: %d.%m.%Y - %H:%M:%S")
+        updated_at = parse_date(date_col, "Stand: %d.%m.%Y - %H:%M:%S")
 
-    data = {
-        "last_updated": update_time,
-        "lots": []
-    }
+    lots = Lots()
+    for lot_list in soup.find_all("div", {"class": "listing"}):
+        raw_lots = lot_list.select('tr + tr')
 
-    # get all tables with lots
-    raw_lot_list = soup.find_all("div", {"class": "listing"})
+        for raw_lot in raw_lots:
+            name = raw_lot.select('a')[0].text
+            lot = geodata.lot(name)
+            lot.updated_at = updated_at
+            lot.free = int(raw_lot.select('td + td')[0].text)
 
-    # get all lots
-    for lot_list in raw_lot_list:
-        raw_lots = lot_list.select('tr + tr')
+            if "green" in str(raw_lot.select("td + td")[0]):
+                lot.state = "open"
+            else:
+                lot.state = "closed"
 
-        for lot in raw_lots:
-            lot_name = lot.select('a')[0].text
-
-            try:
-                lot_free = int(lot.select('td + td')[0].text)
-            except ValueError:
-                lot_free = 0
-
-            try:
-                if "green" in str(lot.select("td + td")[0]):
-                    lot_state = "open"
-                else:
-                    lot_state = "closed"
-            except ValueError:
-                lot_state = "nodata"
-
-            lot = geodata.lot(lot_name)
-            data["lots"].append({
-                "name": lot_name,
-                "free": lot_free,
-                "total": lot.total,
-                "coords": lot.coords,
-                "state": lot_state,
-                "id": lot.id,
-                "forecast": False
-            })
-
-    return data
+            lots.append(lot)
+    return lots