Merge pull request #673 from azavea/feature/kjh/write-tiles-to-cache

Write tiles to S3 cache

common.yml

@@ -18,6 +18,7 @@ services:
       - PFB_AWS_BATCH_TILEMAKER_JOB_DEFINITION_NAME_REVISION
       - PFB_AWS_BATCH_TILEMAKER_JOB_DEFINITION_NAME
       - PFB_TILEGARDEN_ROOT=http://localhost:9400
+      - PFB_TILEGARDEN_CACHE_BUCKET=dev-pfb-tilecache-us-east-1
     volumes:
       - $HOME/.aws:/root/.aws:ro
 

scripts/clear-tile-cache

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -e
+
+DEFAULT_BUCKET_NAME=$(docker-compose run --rm tilegarden bash -c 'echo -n $PFB_TILEGARDEN_CACHE_BUCKET')
+
+function usage() {
+    echo -n \
+         "Usage: $(basename "$0") [--list] [BUCKET]
+
+Removes all cached tiles (that is, all files) from the configured S3 cache bucket:
+    $DEFAULT_BUCKET_NAME
+
+The target bucket can be overridden by passing a different bucket name as an argument.
+
+With the --list option, lists and summarizes the contents before asking for confirmation.
+"
+}
+
+if [ "${BASH_SOURCE[0]}" = "${0}" ]
+then
+    if [ "${1:-}" = "--help" ]; then
+        usage
+        exit
+    fi
+
+    if [ "${1:-}" = "--list" ]; then
+        SHOW_LIST=true
+        shift
+    fi
+
+    BUCKET_NAME="${1:-$DEFAULT_BUCKET_NAME}"
+
+    if [ $SHOW_LIST ]; then
+        aws s3 ls --recursive --summarize "s3://${BUCKET_NAME}"
+        echo ""
+    fi
+
+    echo "This will delete all files from the '$BUCKET_NAME' S3 bucket."
+
+    read -r -p "Are you sure? [y/N] " response
+    response=${response,,}    # tolower
+    if [[ "$response" =~ ^(yes|y)$ ]]
+    then
+        aws s3 rm --recursive "s3://${BUCKET_NAME}"
+    fi
+fi

src/tilegarden/.env.dev

@@ -1,5 +1,7 @@
+AWS_PROFILE
 PFB_DB_HOST=database.service.pfb.internal
 PFB_DB_DATABASE=pfb
 PFB_DB_PASSWORD=pfb
 PFB_DB_PORT=5432
 PFB_DB_USER=pfb
+PFB_TILEGARDEN_CACHE_BUCKET=dev-pfb-tilecache-us-east-1

src/tilegarden/.eslintrc.json

@@ -4,15 +4,17 @@
   },
   "extends": "airbnb-base",
   "rules": {
-    "semi": [2, "never"],
-    "no-unexpected-multiline": 2,
+    "camelcase": 2,
     "indent": ["error", 4],
+    "import/no-extraneous-dependencies": ["error", {"optionalDependencies": true}],
+    "no-console": 0,
+    "no-unexpected-multiline": 2,
     "max-len": ["error", 100, { "ignoreComments": true}],
+    "max-statements": ["error", 50],
+    "object-curly-newline": ["error", { "consistent": true }],
     "quotes": ["error", "single"],
-    "camelcase": 2,
-    "vars-on-top": 2,
+    "semi": [2, "never"],
     "strict": 2,
-    "max-statements": ["error", 50],
-    "import/no-extraneous-dependencies": ["error", {"optionalDependencies": true}]
+    "vars-on-top": 2
   }
 }

src/tilegarden/src/api.js

@@ -3,6 +3,7 @@
  */
 
 const APIBuilder = require('claudia-api-builder')
+const aws = require('aws-sdk')
 
 const { imageTile, createMap } = require('./tiler')
 const HTTPError = require('./util/error-builder')
@@ -16,11 +17,11 @@ const HTML_RESPONSE = { success: { contentType: 'text/html' } }
 // Converts a req object to a set of coordinates
 const processCoords = (req) => {
     // Handle url params
-    const z = Number(req.pathParams.z)
-    const x = Number(req.pathParams.x)
+    const z = Number(req.pathParameters.z)
+    const x = Number(req.pathParameters.x)
 
     // strip .png off of y if necessary
-    const preY = req.pathParams.y
+    const preY = req.pathParameters.y
     const y = Number(preY.substr(0, preY.lastIndexOf('.')) || preY)
 
     // Check type of coords
@@ -33,15 +34,17 @@ const processCoords = (req) => {
 
 const getPositionalFilters = (req) => {
     /* eslint-disable-next-line object-curly-newline */
-    const { x, y, z, config, ...remainder } = req.pathParams
+    const { x, y, z, config, ...remainder } = req.pathParameters
     return remainder
 }
 
 // Returns a properly formatted list of layers
 // or an empty list if there are none
 const processLayers = (req) => {
-    if (req.queryString.layers) return JSON.parse(req.queryString.layers)
-    else if (req.queryString.layer || req.queryString.filter || req.queryString.filters) {
+    if (req.queryStringParameters.layers) return JSON.parse(req.queryStringParameters.layers)
+    else if (req.queryStringParameters.layer ||
+             req.queryStringParameters.filter ||
+             req.queryStringParameters.filters) {
         /* eslint-disable-next-line quotes */
         throw HTTPError("Invalid argument, did you mean '&layers='?", 400)
     }
@@ -51,12 +54,12 @@ const processLayers = (req) => {
 
 // Parses out the configuration specifications
 const processConfig = req => ({
-    s3bucket: req.queryString.s3bucket,
-    config: req.pathParams.config,
+    s3bucket: req.queryStringParameters.s3bucket,
+    config: req.pathParameters.config,
 })
 
 // Create new lambda API
-const api = new APIBuilder()
+const api = new APIBuilder({ requestFormat: 'AWS_PROXY' })
 
 // Handles error by returning an API response object
 const handleError = (e) => {
@@ -69,6 +72,41 @@ const handleError = (e) => {
     )
 }
 
+/* Uploads a tile to the S3 cache, using its request path as a key
+ *
+ * Does nothing unless there's a CACHE_BUCKET set in the environment.
+ * Returns the Promise<tile> again for chaining.
+ *
+ * Theoretically it should be possible to run the upload in parallel and not make the request
+ * wait for it before returning the tile, but in fact the process gets killed when the main promise
+ * resolves so the upload doesn't manage to finish.
+ */
+const writeToS3 = (tile, req) => {
+    const s3CacheBucket = process.env.PFB_TILEGARDEN_CACHE_BUCKET
+    if (s3CacheBucket) {
+        let key = req.path
+        // API Gateway includes a 'path' property but claudia-local-api currently doesn't
+        // (see https://github.com/azavea/claudia-local-api/issues/1), so this reconstructs it.
+        if (!key) {
+            /* eslint-disable camelcase */
+            const { z, x, y, job_id, config } = req.pathParameters
+            key = `tile/${job_id}/${config}/${z}/${x}/${y}`
+            /* eslint-enable camelcase */
+        }
+
+        const upload = new aws.S3().putObject({
+            Bucket: s3CacheBucket,
+            Key: key,
+            Body: tile,
+        })
+        return upload.promise().then(() => {
+            console.debug(`Uploaded tile to S3: ${key}`)
+            return tile
+        })
+    }
+    return new Promise(resolve => resolve(tile))
+}
+
 // Get tile for some zxy bounds
 api.get(
     '/tile/{job_id}/{config}/{z}/{x}/{y}',
@@ -80,6 +118,7 @@ api.get(
             const configOptions = processConfig(req)
 
             return imageTile(createMap(z, x, y, filters, layers, configOptions))
+                .then(tile => writeToS3(tile, req))
                 .then(img => new APIBuilder.ApiResponse(img, IMAGE_HEADERS, 200))
                 .catch(handleError)
         } catch (e) {

src/tilegarden/tests/api.test.js

@@ -6,7 +6,7 @@ const api = rewire('../src/api'),
 describe('processCoords', () => {
     test('properly parses properly formatted coords', () => {
         const req = {
-            pathParams: {
+            pathParameters: {
                 x: '2385',
                 y: '3103.png',
                 z: '13'
@@ -21,14 +21,14 @@ describe('processCoords', () => {
 
     test('parses extensions other than png', () => {
         const req = {
-            pathParams: {
+            pathParameters: {
                 x: '2385',
                 y: '3103.jpg',
                 z: '13'
             }
         }
         const req2 = {
-            pathParams: {
+            pathParameters: {
                 x: '2385',
                 y: '3103.pdf',
                 z: '13'
@@ -48,7 +48,7 @@ describe('processCoords', () => {
 
     test('properly parses coords with no file extension', () => {
         const req = {
-            pathParams: {
+            pathParameters: {
                 x: '2385',
                 y: '3103',
                 z: '13'
@@ -63,7 +63,7 @@ describe('processCoords', () => {
 
     test('throws an error for non-numeral coords', () => {
         const req = {
-            pathParams: {
+            pathParameters: {
                 x: 'eggs',
                 y: 'bacon',
                 z: 'orange_juice'
@@ -76,7 +76,7 @@ describe('processCoords', () => {
 describe('processLayers', () => {
     test('properly parses list of layer', () => {
         const req = {
-            queryString: {
+            queryStringParameters: {
                 layers: '["layer1","layer2","layer3","layer4"]'
             }
         }
@@ -85,7 +85,7 @@ describe('processLayers', () => {
 
     test('properly parses just one layer', () => {
         const req = {
-            queryString: {
+            queryStringParameters: {
                 layers: '["layer"]'
             }
         }
@@ -94,14 +94,14 @@ describe('processLayers', () => {
 
     test('properly parses no fields', () => {
         const req = {
-            queryString: {}
+            queryStringParameters: {}
         }
         expect(processLayers(req)).toEqual([])
     })
 
     test('properly parses a blank field', () => {
         const req = {
-            queryString: {
+            queryStringParameters: {
                 layers: ''
             }
         }