From c13c50fd34ed00045b8f112c1b917c2d961d3f06 Mon Sep 17 00:00:00 2001 From: Christian Landgren <christian@landgren.nu> Date: Wed, 10 Aug 2022 15:41:59 +0200 Subject: [PATCH] Feat/opentiles (draft) (#96) * feat: add dependencies - osrm, vroom, pelias, to stack * chore: add missing namespaces * chore: make dependencies optional * chore: limit resources to prevent these services from grabbing all cpu on the nodes * chore: restore csv importer from test * chore: fix correct ingress * feat: use manifests, not kustomize for dependencies. Also liveness probe * feat: add open tiles --- k8s/dependencies/osrm/Dockerfile | 10 ++ k8s/dependencies/osrm/osrm.yaml | 103 ++++++++++++++++ k8s/dependencies/tiles/opentiles.yaml | 107 +++++++++++++++++ k8s/dependencies/vroom/vroom.yaml | 167 ++++++++++++++++++++++++++ skaffold.yaml | 2 + 5 files changed, 389 insertions(+) create mode 100644 k8s/dependencies/osrm/Dockerfile create mode 100644 k8s/dependencies/osrm/osrm.yaml create mode 100644 k8s/dependencies/tiles/opentiles.yaml create mode 100644 k8s/dependencies/vroom/vroom.yaml diff --git a/k8s/dependencies/osrm/Dockerfile b/k8s/dependencies/osrm/Dockerfile new file mode 100644 index 00000000..dcdd5866 --- /dev/null +++ b/k8s/dependencies/osrm/Dockerfile @@ -0,0 +1,10 @@ +FROM osrm/osrm-backend:v5.25.0 +ENV DOWNLOAD_URL=https://download.geofabrik.de/europe/sweden-latest.osm.pbf +RUN apt-get update && apt-get install -y --no-install-recommends wget tar && \ + mkdir /data && \ + wget --no-check-certificate --debug "${DOWNLOAD_URL}" -O /data/sweden-latest.osm.pbf && \ + rm -rf /var/lib/apt/lists/* && \ + osrm-extract -p /opt/car.lua /data/sweden-latest.osm.pbf && \ + osrm-partition /data/sweden-latest.osrm && \ + osrm-customize /data/sweden-latest.osrm && \ + osrm-routed --algorithm mld --max-table-size 10000 /data/sweden-latest.osrm \ No newline at end of file diff --git a/k8s/dependencies/osrm/osrm.yaml b/k8s/dependencies/osrm/osrm.yaml new file mode 100644 index 00000000..cf8a6a27 --- /dev/null +++ b/k8s/dependencies/osrm/osrm.yaml @@ -0,0 +1,103 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: osrm + namespace: osrm +spec: + replicas: 3 + selector: + matchLabels: + app: osrm + serviceName: osrm + template: + metadata: + labels: + app: osrm + spec: + containers: + - image: osrm/osrm-backend:v5.25.0 + name: osrm + command: ["/bin/sh"] + args: + - -c + - | + apt-get update && apt-get install -y --no-install-recommends wget tar && \ + wget --no-check-certificate --debug "${DOWNLOAD_URL}" -O /data/sweden-latest.osm.pbf && \ + rm -rf /var/lib/apt/lists/* && \ + osrm-extract -p /opt/car.lua /data/sweden-latest.osm.pbf && \ + osrm-partition /data/sweden-latest.osrm && \ + osrm-customize /data/sweden-latest.osrm && \ + osrm-routed --algorithm mld --max-table-size 10000 /data/sweden-latest.osrm + env: + - name: DOWNLOAD_URL + value: https://download.geofabrik.de/europe/sweden-latest.osm.pbf + resources: + requests: + cpu: "0.3" + memory: "512Mi" + ports: + - containerPort: 5000 + protocol: TCP + volumeMounts: + - name: osrm-data + mountPath: "/data" + livenessProbe: + initialDelaySeconds: 500 + httpGet: + path: /nearest/v1/driving/21.033499654770427,64.74958466530941 + port: 5000 + readinessProbe: + initialDelaySeconds: 100 + httpGet: + path: /nearest/v1/driving/21.033499654770427,64.74958466530941 + port: 5000 + volumeClaimTemplates: + - metadata: + name: osrm-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "10Gi" +--- +apiVersion: v1 +kind: Service +metadata: + name: osrm + namespace: osrm + labels: + app: osrm +spec: + ports: + - name: http + protocol: TCP + port: 5000 + targetPort: 5000 + selector: + app: osrm +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: osrm-ingress + namespace: osrm + annotations: + kubernetes.io/ingress.class: "nginx" + cert-manager.io/cluster-issuer: "letsencrypt-prod" +spec: + tls: + - hosts: + - osrm.iteam.services + secretName: osrm-prod-tls + rules: + - host: osrm.iteam.services + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: osrm + port: + number: 5000 \ No newline at end of file diff --git a/k8s/dependencies/tiles/opentiles.yaml b/k8s/dependencies/tiles/opentiles.yaml new file mode 100644 index 00000000..29d5ffb6 --- /dev/null +++ b/k8s/dependencies/tiles/opentiles.yaml @@ -0,0 +1,107 @@ +# docker run \ +# -e DOWNLOAD_PBF=https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf \ +# -e DOWNLOAD_POLY=https://download.geofabrik.de/europe/luxembourg.poly \ +# -e ALLOW_CORS=enabled \ +# -v osm-data:/data/database/ \ +# overv/openstreetmap-tile-server \ +# import + + + +# --- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: opentiles + namespace: opentiles +spec: + replicas: 1 + selector: + matchLabels: + app: opentiles + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: opentiles + app-group: opentiles + spec: + containers: + - name: opentiles + image: overv/openstreetmap-tile-server + args: + - "import" + volumeMounts: + - name: opentiles + mountPath: /data/database + env: + - name: DOWNLOAD_PBF + value: https://download.geofabrik.de/europe/sweden-latest.osm.pbf + - name: DOWNLOAD_POLY + value: https://download.geofabrik.de/europe/sweden.poly + - name: ALLOW_CORS + value: enabled + resources: + requests: + memory: "0.25Gi" + cpu: "0.1" + volumes: + - name: opentiles + persistentVolumeClaim: + claimName: opentiles + +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: opentiles + namespace: opentiles +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: opentiles + namespace: opentiles +spec: + selector: + app-group: opentiles + ports: + - name: http + protocol: TCP + port: 80 + type: ClusterIP +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: opentiles-ingress + namespace: opentiles + annotations: + kubernetes.io/ingress.class: "nginx" + cert-manager.io/cluster-issuer: "letsencrypt-prod" + +spec: + tls: + - hosts: + - opentiles.iteam.services + secretName: opentiles-prod-tls + rules: + - host: opentiles.iteam.services + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: opentiles + port: + number: 80 diff --git a/k8s/dependencies/vroom/vroom.yaml b/k8s/dependencies/vroom/vroom.yaml new file mode 100644 index 00000000..4651b26c --- /dev/null +++ b/k8s/dependencies/vroom/vroom.yaml @@ -0,0 +1,167 @@ +apiVersion: v1 +kind: Service +metadata: + name: vroom + namespace: vroom +spec: + ports: + - port: 3000 + targetPort: 3000 + selector: + app: vroom +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vroom + namespace: vroom +spec: + selector: + matchLabels: + app: vroom + template: + metadata: + labels: + app: vroom + spec: + containers: + - name: vroom + image: vroomvrp/vroom-docker:v1.11.0 + ports: + - containerPort: 3000 + resources: + requests: + cpu: "0.3" + memory: "512Mi" + limits: + cpu: "1.5" + memory: "2Gi" + env: + - name: VROOM_ROUTER + value: osrm + - name: OSRM_URL + value: osrm + volumeMounts: + - name: vroom + mountPath: /vroom-express/config.yml + subPath: config.yml + volumes: + - name: vroom + configMap: + items: + - key: config.yml + path: config.yml + name: vroom +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: vroom-ingress + namespace: vroom + annotations: + kubernetes.io/ingress.class: "nginx" + cert-manager.io/cluster-issuer: "letsencrypt-prod" +spec: + tls: + - hosts: + - vroom.iteam.services + secretName: vroom-prod-tls + rules: + - host: vroom.iteam.services + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: vroom + port: + number: 3000 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vroom + namespace: vroom +data: + config.yml: |- + cliArgs: + geometry: false # retrieve geometry (-g) + planmode: false # run vroom in plan mode (-c) if set to true + threads: 4 # number of threads to use (-t) + explore: 5 # exploration level to use (0..5) (-x) + limit: '1mb' # max request size + logdir: '/..' # the path for the logs relative to ./src + logsize: '100M' # max log file size for rotation + maxlocations: 1000 # max number of jobs/shipments locations + maxvehicles: 200 # max number of vehicles + override: true # allow cli options override (-c, -g, -t and -x) + path: '' # VROOM path (if not in $PATH) + port: 3000 # expressjs port + router: 'osrm' # routing backend (osrm, libosrm or ors) + timeout: 300000 # milli-seconds + baseurl: '/' #base url for api + routingServers: + osrm: + car: + host: 'osrm.osrm.svc.cluster.local' + port: '5000' + bike: + host: 'osrm.osrm.svc.cluster.local' + port: '5000' + foot: + host: 'osrm.osrm.svc.cluster.local' + port: '5000' + ors: + driving-car: + host: '0.0.0.0' + port: '8080' + driving-hgv: + host: '0.0.0.0' + port: '8080' + cycling-regular: + host: '0.0.0.0' + port: '8080' + cycling-mountain: + host: '0.0.0.0' + port: '8080' + cycling-road: + host: '0.0.0.0' + port: '8080' + cycling-electric: + host: '0.0.0.0' + port: '8080' + foot-walking: + host: '0.0.0.0' + port: '8080' + foot-hiking: + host: '0.0.0.0' + port: '8080' + valhalla: + auto: + host: '0.0.0.0' + port: '8002' + bicycle: + host: '0.0.0.0' + port: '8002' + pedestrian: + host: '0.0.0.0' + port: '8002' + motorcycle: + host: '0.0.0.0' + port: '8002' + motor_scooter: + host: '0.0.0.0' + port: '8002' + taxi: + host: '0.0.0.0' + port: '8002' + hov: + host: '0.0.0.0' + port: '8002' + truck: + host: '0.0.0.0' + port: '8002' + bus: + host: '0.0.0.0' + port: '8002' diff --git a/skaffold.yaml b/skaffold.yaml index df6a3e4b..e00150dc 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -4,6 +4,8 @@ metadata: name: pm-digital-twin build: artifacts: + - image: iteam1337/osrm-sweden + context: k8s/dependencies/osrm - image: iteam1337/pm-digital-twin-simulation-engine context: packages/simulator - image: iteam1337/pm-digital-twin-visualisation