WIP: stats: Add support for graph generation

roles/stats/defaults/main.yml

@@ -35,3 +35,16 @@ stats_fastly_addresses:
   - "202.21.128.12/32"
   - "203.57.145.11/32"
   - "203.57.145.12/32"
+
+# Cache folder of the stats graph generation
+# FIXME: might not need stats ...
+stats_graph_cache_dir: /var/cache/flathub-stats-graph
+
+# Where the code for the stats graph generation is stored
+stats_graph_checkout_dir: /opt/flathub-stats-graph
+
+# Where the output of the graphs is put
+stats_graph_output_dir: '{{ stats_volume_mount }}/graphs'
+
+# What the web path to the graphs folder appears to the outside
+stats_graph_web_path: /graphs

roles/stats/tasks/graph.yml

@@ -0,0 +1,57 @@
+---
+
+# FIXME: do we need tags ?
+
+# Dependencies
+
+# FIXME: This is all I install in a debian:stretch-slim container, assume it'll be enough...
+- name: install stats graphs depends
+  package: name='{{ item }}' state=latest
+  with_items:
+    - python3-matplotlib
+    - python3-requests
+
+# Get the code and setup dirs and helpers
+
+# FIXME: how do we update the code?
+# do we have a git pull in the scheduled script ?
+# or do we just redeploy the playbook each time ?
+- name: git clone stats graph generation code
+  git:
+    repo: https://gitlab.com/ahayzen/flathub-api-stats-generator.git
+    dest: '{{ stats_graph_checkout_dir }}'
+
+- name: install stats graph run helper
+  template:
+    src: flathub-stats-graph.sh.j2
+    dest: '{{ stats_graph_checkout_dir }}/flathub-stats-graph.sh'
+    mode: 0744
+
+# FIXME: do we need the SELinux stuff? if we do, probably need elsewhere...
+- name: create stats graph output directory
+  file:
+    path: '{{ stats_graph_output_dir }}'
+    state: directory
+    seuser: _default
+    serole: _default
+    setype: _default
+  tags:
+    - stats
+
+- name: install html overview page
+  template:
+    src: flathub-stats-overview.html.j2
+    dest: '{{ stats_graph_checkout_dir }}/flathub.html'
+
+# Setup systemd service and timer
+
+- name: install flathub stats graph systemd service and timer
+  template:
+    src: '{{ item }}.j2'
+    dest: '/etc/systemd/system/{{ item }}'
+  with_items:
+    - "flathub-stats-graph.service"
+    - "flathub-stats-graph.timer"
+
+- name: start flathub stats graph systemd timer
+  systemd: name="flathub-stats-graph.timer" daemon_reload=yes enabled=yes state=started

roles/stats/tasks/main.yml

@@ -172,3 +172,10 @@
   tags:
     - stats
     - stats_nginx
+
+# Stats Graph generation
+# FIXME: what tags do we need?
+- name: setup graphing of stats
+  include: graph.yml
+  tags:
+    - stats

roles/stats/templates/flathub-stats-graph.service.j2

@@ -0,0 +1,8 @@
+[Unit]
+Description=Daily flathub stats graph update
+
+[Service]
+Type=oneshot
+ExecStart={{ stats_graph_checkout_dir }}/flathub-stats-graph.sh
+KillMode=process
+WorkingDirectory={{ stats_graph_checkout_dir }}

roles/stats/templates/flathub-stats-graph.sh.j2

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# FIXME: should we have a git pull here ?
+
+# FIXME: should we use --cache-dir ?
+# maybe have a special mode which can read from local filesystem rather than using network?
+# or will hitting flathub.org/stats be super fast as that is us ...
+
+# Update graph images
+/usr/bin/python3 {{ stats_graph_checkout_dir }}/main.py \
+  --cache-dir {{ stats_graph_cache_dir }} \
+  --model-skip-today \
+  --report \
+    count_by_time \
+    downloads \
+    downloads_by_arch \
+    downloads_by_flatpak_major_version \
+    downloads_by_flatpak_major_version \
+    downloads_by_flatpak_version \
+    downloads_by_flatpak_version \
+    downloads_by_ostree_version \
+    downloads_by_ostree_version \
+    downloads_only_apps \
+    downloads_only_runtimes \
+    downloads_only_nvidia_runtimes \
+  --type \
+    graph \
+    graph \
+    graph \
+    graph \
+    graph:stacked_bar \
+    graph \
+    graph:stacked_bar \
+    graph \
+    graph:stacked_bar \
+    graph \
+    graph \
+    data \
+  --output \
+    {{ stats_graph_output_dir }}/flathub.png \
+    {{ stats_graph_output_dir }}/flathub_downloads.png \
+    {{ stats_graph_output_dir }}/flathub_downloads_by_arch.png \
+    {{ stats_graph_output_dir }}/flathub_downloads_by_flatpak_major_version.png \
+    {{ stats_graph_output_dir }}/flathub_downloads_by_flatpak_major_version_stacked_bar.png \
+    {{ stats_graph_output_dir }}/flathub_downloads_by_flatpak_version.png \
+    {{ stats_graph_output_dir }}/flathub_downloads_by_flatpak_version_stacked_bar.png \
+    {{ stats_graph_output_dir }}/flathub_downloads_by_ostree_version.png \
+    {{ stats_graph_output_dir }}/flathub_downloads_by_ostree_version_stacked_bar.png \
+    {{ stats_graph_output_dir }}/flathub_downloads_only_apps.png \
+    {{ stats_graph_output_dir }}/flathub_downloads_only_runtimes.png \
+    {{ stats_graph_output_dir }}/flathub_downloads_only_nvidia_runtimes.txt

roles/stats/templates/flathub-stats-graph.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Daily flathub stats graph update
+
+[Timer]
+OnCalendar=*-*-* 09:00
+RandomizedDelaySec=10m
+Persistent=true
+
+[Install]
+WantedBy=timers.target

roles/stats/templates/flathub-stats-overview.html.j2

@@ -0,0 +1,63 @@
+<!doctype html>
+<html>
+<head>
+    <title>Flathub Statistics</title>
+</head>
+<body style="text-align: center;">
+    <h1>Flathub Statistics</h1>
+    <p>Please report bugs to <a href="https://gitlab.com/ahayzen/flathub-api-stats-generator/issues">https://gitlab.com/ahayzen/flathub-api-stats-generator/issues</a></p>
+
+    <h2>Notices</h2>
+    <p>Please note that there was data loss in the download charts for the dates, 2018-06-18, 2018-06-19, 2018-06-20, 2018-07-24, 2018-08-14, 2018-11-19 -> 2018-11-28, 2018-12-08 -> 2018-12-11</p>
+
+    <h2>Contents</h2>
+    <ul>
+      <li><a href="#appcount">App Count</a></li>
+      <li><a href="#downloads">Downloads</a></li>
+      <li><a href="#downloadsonlyapps">Downloads (only apps)</a></li>
+      <li><a href="#downloadsonlyruntimes">Downloads (only runtimes)</a></li>
+      <li><a href="#downloadsbyarch">Downloads by arch</a></li>
+      <li><a href="#downloadsbyflatpak">Downloads by flatpak version</a></li>
+      <li><a href="#downloadsbyflatpakstacked">Downloads by flatpak version (stacked percentage)</a></li>
+      <li><a href="#downloadsbyostree">Downloads by ostree version</a></li>
+      <li><a href="#downloadsbyostreestacked">Downloads by ostree version (stacked percentage)</a></li>
+      <li><a href="#downloadsbyflatpakmajor">Downloads by flatpak major version</a></li>
+      <li><a href="#downloadsbyflatpakmajorstacked">Downloads by flatpak major version (stacked percentage)</a></li>
+      <li><a href="{{ stats_graph_web_path }}/flathub_downloads_only_nvidia_runtimes.txt">Downloads by nvidia runtime (text output on another page)</a></li>
+    </ul>
+
+    <h2 id="appcount">App count</h2>
+    <img src="{{ stats_graph_web_path }}/flathub.png"/>
+
+    <h2 id="downloads">Downloads</h2>
+    <img src="{{ stats_graph_web_path }}/flathub_downloads.png"/>
+
+    <h2 id="downloadsonlyapps">Downloads (only apps)</h2>
+    <img src="{{ stats_graph_web_path }}/flathub_downloads_only_apps.png"/>
+
+    <h2 id="downloadsonlyruntimes">Downloads (only runtimes)</h2>
+    <img src="{{ stats_graph_web_path }}/flathub_downloads_only_runtimes.png"/>
+
+    <h2 id="downloadsbyarch">Downloads by arch</h2>
+    <img src="{{ stats_graph_web_path }}/flathub_downloads_by_arch.png"/>
+
+    <h2 id="downloadsbyflatpak">Downloads by flatpak version</h2>
+    <img src="{{ stats_graph_web_path }}/flathub_downloads_by_flatpak_version.png"/>
+
+    <h2 id="downloadsbyflatpakstacked">Downloads by flatpak version (stacked percentage)</h2>
+    <img src="{{ stats_graph_web_path }}/flathub_downloads_by_flatpak_version_stacked_bar.png"/>
+
+    <h2 id="downloadsbyostree">Downloads by ostree version</h2>
+    <img src="{{ stats_graph_web_path }}/flathub_downloads_by_ostree_version.png"/>
+
+    <h2 id="downloadsbyostreestacked">Downloads by ostree version (stacked percentage)</h2>
+    <img src="{{ stats_graph_web_path }}/flathub_downloads_by_ostree_version_stacked_bar.png"/>
+
+    <h2 id="downloadsbyflatpakmajor">Downloads by flatpak major version</h2>
+    <img src="{{ stats_graph_web_path }}/flathub_downloads_by_flatpak_major_version.png"/>
+
+    <h2 id="downloadsbyflatpakmajorstacked">Downloads by flatpak major version (stacked percentage)</h2>
+    <img src="{{ stats_graph_web_path }}/flathub_downloads_by_flatpak_major_version_stacked_bar.png"/>
+</body>
+</html>
+