Add benchmarks

.github/workflows/action.yml

@@ -2,10 +2,6 @@ name: tests
 
 on: [push]
 
-env:
-    AWS_ACCESS_KEY_ID: "FOO"
-    AWS_SECRET_ACCESS_KEY: "BAR"
-
 jobs:
   test:
     runs-on: ${{ matrix.os }}
@@ -14,6 +10,7 @@ jobs:
         rust: [stable]
         os: [ubuntu-latest]
     steps:
+      # Run tests
       - uses: actions/checkout@v2
       - name: Install Rust
         uses: actions-rs/toolchain@v1
@@ -22,12 +19,6 @@ jobs:
             override: true
             components: rustfmt, clippy
 
-      - name: Install cargo-lambda
-        uses: actions-rs/cargo@v1
-        with:
-          command: install
-          args: cargo-lambda
-
       - uses: Swatinem/rust-cache@v1
       - name: Run cargo fmt
         uses: actions-rs/cargo@v1
@@ -62,4 +53,20 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --all-targets --all-features
+          args: --all-features
+
+      # Run benchmarks
+      - name: Run benchmarks
+        run: cargo bench --bench search-benchmarks --bench request-benchmarks -- LIGHT --output-format bencher | tee output.txt
+      - name: Download previous benchmark data
+        uses: actions/cache@v1
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'cargo'
+          output-file-path: output.txt
+          external-data-json-path: ./cache/benchmark-data.json
+          fail-on-alert: false

.gitignore

@@ -1,7 +1,8 @@
 target
 Cargo.lock
 *.code-workspace
+*.vcf.gz
 .vscode
 .idea
 package-lock.json
-deploy/.build
+deploy/.build

data/events/event_get.json

@@ -1,6 +1,6 @@
 {
   "httpMethod": "GET",
-  "path": "/variants/data/vcf/sample1-bcbio-cancer",
+  "path": "/variants/vcf/sample1-bcbio-cancer",
   "body": null,
 
   "resource": "/{proxy+}",

data/events/event_parameterized_get.json

@@ -1,6 +1,6 @@
 {
   "httpMethod": "GET",
-  "path": "/variants/data/vcf/sample1-bcbio-cancer",
+  "path": "/variants/vcf/sample1-bcbio-cancer",
   "body": null,
 
   "resource": "/{proxy+}",

data/events/event_parameterized_post.json

@@ -1,6 +1,6 @@
 {
   "httpMethod": "POST",
-  "path": "/variants/data/vcf/sample1-bcbio-cancer",
+  "path": "/variants/vcf/sample1-bcbio-cancer",
   "body": "{\"format\": \"VCF\", \"regions\": [{\"referenceName\": \"chrM\"}]}",
 
   "resource": "/{proxy+}",

data/events/event_parameterized_post_class_header.json

@@ -1,6 +1,6 @@
 {
   "httpMethod": "POST",
-  "path": "/variants/data/vcf/sample1-bcbio-cancer",
+  "path": "/variants/vcf/sample1-bcbio-cancer",
   "body": "{\"format\": \"VCF\", \"class\": \"header\", \"regions\": [{\"referenceName\": \"chrM\"}]}",
 
   "resource": "/{proxy+}",

data/events/event_post.json

@@ -1,6 +1,6 @@
 {
   "httpMethod": "POST",
-  "path": "/variants/data/vcf/sample1-bcbio-cancer",
+  "path": "/variants/vcf/sample1-bcbio-cancer",
   "body": null,
 
   "resource": "/{proxy+}",

htsget-config/src/config.rs

@@ -9,13 +9,13 @@ use crate::config::StorageType::LocalStorage;
 use crate::regex_resolver::RegexResolver;
 
 pub const USAGE: &str = r#"
-This executable doesn't use command line arguments, but there are some environment variables that can be set to configure the HtsGet server:
-* HTSGET_ADDR: The socket address to use for the server which creates response tickets. Default: "127.0.0.1:8080".
+The HtsGet server executables don't use command line arguments, but there are some environment variables that can be set to configure them:
+* HTSGET_ADDR: The socket address for the server which creates response tickets. Default: "127.0.0.1:8080".
 * HTSGET_PATH: The path to the directory where the server should be started. Default: ".". Unused if HTSGET_STORAGE_TYPE is "AwsS3Storage".
 * HTSGET_REGEX: The regular expression that should match an ID. Default: ".*".
 For more information about the regex options look in the documentation of the regex crate(https://docs.rs/regex/).
 * HTSGET_SUBSTITUTION_STRING: The replacement expression. Default: "$0".
-* HTSGET_STORAGE_TYPE: Either LocalStorage or AwsS3Storage. Default: "LocalStorage".
+* HTSGET_STORAGE_TYPE: Either "LocalStorage" or "AwsS3Storage", representing which storage type to use. Default: "LocalStorage".
 
 The following options are used for the ticket server.
 * HTSGET_TICKET_SERVER_ADDR: The socket address to use for the server which responds to tickets. Default: "127.0.0.1:8081". Unused if HTSGET_STORAGE_TYPE is not "LocalStorage".

htsget-devtools/Cargo.toml

@@ -9,4 +9,4 @@ edition = "2021"
 serde = { version = "~1.0", features = ["derive"] }
 serde_yaml = "~0.8"
 
-noodles = { version = "0.18.0", features = ["bam", "bcf", "bgzf", "cram", "csi", "sam", "tabix", "vcf"] }
+noodles = { version = "0.18.0", features = ["bam", "bcf", "bgzf", "cram", "csi", "sam", "tabix", "vcf"] }

htsget-http-actix/Cargo.toml

@@ -20,7 +20,17 @@ futures = { version = "0.3" }
 tokio = { version = "1.17", features = ["full"] }
 tracing-actix-web = "0.5"
 tracing = "0.1"
+tracing-subscriber = "0.3"
 
 [dev-dependencies]
-htsget-test-utils = { path = "../htsget-test-utils", default-features = false }
-async-trait = "0.1"
+htsget-test-utils = { path = "../htsget-test-utils", features = ["server-tests"], default-features = false }
+async-trait = "0.1"
+
+criterion = { version = "0.3", features = ["async_tokio"] }
+reqwest = { version = "0.11", features = ["json", "blocking", "rustls-tls"] }
+tempfile = "3.3"
+
+[[bench]]
+name = "request-benchmarks"
+harness = false
+path = "benches/request_benchmarks.rs"

htsget-http-actix/README.md

@@ -3,45 +3,53 @@ This crate should allow to setup an [htsget](http://samtools.github.io/hts-specs
 
 ## Quickstart 
 
-These are some examples with [curl](https://github.com/curl/curl). **For the curl examples shown below to work, we assume that the server is being started from the root of the [htsget-rs project](https://github.com/umccr/htsget-rs)**, so we can use the example files inside the `data` directory.
+These are some examples with [curl](https://github.com/curl/curl). **For the curl examples shown 
+below to work, we assume that the server is being started from the root of
+the [htsget-rs project](https://github.com/umccr/htsget-rs)**, and `HTSGET_PATH="data/"`.
 
-To test them you can run:
+The htsget-http-actix server also requires pem formatted X.509 certificates to access the response tickets.
+
+For example, to generate self-signed certificates, run:
 
 ```shell
-$ cargo run -p htsget-http-actix
+$ openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -sha256 -days 365 -nodes -subj '/CN=localhost'
 ```
 
-From **the top of the project**. Alternatively, the `HTSGET_PATH` environment variable can be set accordingly if the current working directory is `htsget-http-actix`, i.e:
+To test the curl example below, run:
 
 ```shell
-$ HTSGET_PATH=../ cargo run
+$ cargo run -p htsget-http-actix
 ```
 
-Otherwise we could have problems as [directory traversal](https://en.wikipedia.org/wiki/Directory_traversal_attack) isn't allowed.
+
 
 ## Environment variables 
 
 There are reasonable defaults to allow the user to spin up the server as fast as possible, but all of them are configurable via environment variables.
 
 Since this service can be used in serverless environments, no `dotenv` configuration is needed, [adjusting the environment variables below prevent accidental leakage of settings and sensitive information](https://medium.com/@softprops/configuration-envy-a09584386705).
 
-| Variable | Description | Default |
-|---|---|---|
-| HTSGET_IP| IP address | 127.0.0.1 |
-| HTSGET_PORT| TCP Port | 8080 |
-| HTSGET_PATH| The path to the directory where the server starts | `$PWD` | 
-| HTSGET_REGEX| The regular expression an ID should match. | ".*" |
-| HTSGET_REPLACEMENT| The replacement expression, to produce a key from an ID. | "$0" |
-| HTSGET_ID| ID of the service. | "" |
-| HTSGET_NAME| Name of the service. | HtsGet service |
-| HTSGET_VERSION | Version of the service | ""
-| HTSGET_ORGANIZATION_NAME| Name of the organization | Snake oil
-| HTSGET_ORGANIZATION_URL| URL of the organization | https://en.wikipedia.org/wiki/Snake_oil |
-| HTSGET_CONTACT_URL | URL to provide contact to the users | "" |
-| HTSGET_DOCUMENTATION_URL| Link to documentation | https://github.com/umccr/htsget-rs/tree/main/htsget-http-actix |
-| HTSGET_CREATED_AT | Date of the creation of the service. | "" |
-| HTSGET_UPDATED_AT | Date of the last update of the service. | "" |
-| HTSGET_ENVIRONMENT | Environment in which the service is running. | Testing |
+| Variable                   | Description                                                                                                                              | Default          |
+|----------------------------|------------------------------------------------------------------------------------------------------------------------------------------|------------------|
+| HTSGET_ADDR                | The socket address for the server which creates response tickets.                                                                        | "127.0.0.1:8080" |
+| HTSGET_PATH                | The path to the directory where the server starts                                                                                        | "."              | 
+| HTSGET_REGEX               | The regular expression an ID should match.                                                                                               | ".*"             |
+| HTSGET_SUBSTITUTION_STRING | The replacement expression, to produce a key from an ID.                                                                                 | "$0"             |
+| HTSGET_STORAGE_TYPE        | Either "LocalStorage" or "AwsS3Storage", representing which storage type to use.                                                         | "LocalStorage"   |
+| HTSGET_TICKET_SERVER_ADDR  | The socket address to use for the server which responds to tickets. Unused if HTSGET_STORAGE_TYPE is not "LocalStorage".                 | "127.0.0.1:8081" |
+| HTSGET_TICKET_SERVER_KEY   | The path to the PEM formatted X.509 private key used by the ticket response server. Unused if HTSGET_STORAGE_TYPE is not "LocalStorage". | "key.pem"        |
+| HTSGET_TICKET_SERVER_CERT  | The path to the PEM formatted X.509 certificate used by the ticket response server. Unused if HTSGET_STORAGE_TYPE is not "LocalStorage". | "cert.pem"       |
+| HTSGET_S3_BUCKET           | The name of the AWS S3 bucket. Unused if HTSGET_STORAGE_TYPE is not "AwsS3Storage".                                                      | ""               |
+| HTSGET_ID                  | ID of the service.                                                                                                                       | "None"           |
+| HTSGET_NAME                | Name of the service.                                                                                                                     | "None"           |
+| HTSGET_VERSION             | Version of the service.                                                                                                                  | "None"           |
+| HTSGET_ORGANIZATION_NAME   | Name of the organization.                                                                                                                | "None"           |
+| HTSGET_ORGANIZATION_URL    | URL of the organization.                                                                                                                 | "None"           |
+| HTSGET_CONTACT_URL         | URL to provide contact to the users.                                                                                                     | "None"           |
+| HTSGET_DOCUMENTATION_URL   | Link to documentation.                                                                                                                   | "None"           |
+| HTSGET_CREATED_AT          | Date of the creation of the service.                                                                                                     | "None"           |
+| HTSGET_UPDATED_AT          | Date of the last update of the service.                                                                                                  | "None"           |
+| HTSGET_ENVIRONMENT         | Environment in which the service is running.                                                                                             | "None"           |
 For more information about the regex options look in the [documentation of the regex crate](https://docs.rs/regex/).
 
 ## Example cURL requests
@@ -51,25 +59,25 @@ As mentioned above, please keep in mind that the server will take the path where
 ### GET
 
 ```shell
-$ curl '127.0.0.1:8080/variants/data/vcf/sample1-bcbio-cancer'
+$ curl '127.0.0.1:8080/variants/vcf/sample1-bcbio-cancer'
 ```
 
 ### POST
 
 ```shell
-$ curl --header "Content-Type: application/json" -d '{}' '127.0.0.1:8080/variants/data/vcf/sample1-bcbio-cancer'
+$ curl --header "Content-Type: application/json" -d '{}' '127.0.0.1:8080/variants/vcf/sample1-bcbio-cancer'
 ```
 
 ### Parametrised GET
 
 ```shell
-$ curl '127.0.0.1:8080/variants/data/vcf/sample1-bcbio-cancer?format=VCF&class=header'
+$ curl '127.0.0.1:8080/variants/vcf/sample1-bcbio-cancer?format=VCF&class=header'
 ```
 
 ### Parametrised POST
 
 ```shell
-$ curl --header "Content-Type: application/json" -d '{"format": "VCF", "regions": [{"referenceName": "chrM"}]}' '127.0.0.1:8080/variants/data/vcf/sample1-bcbio-cancer'
+$ curl --header "Content-Type: application/json" -d '{"format": "VCF", "regions": [{"referenceName": "chrM"}]}' '127.0.0.1:8080/variants/vcf/sample1-bcbio-cancer'
 ```
 
 ### Service-info
@@ -78,9 +86,29 @@ $ curl --header "Content-Type: application/json" -d '{"format": "VCF", "regions"
 $ curl 127.0.0.1:8080/variants/service-info
 ```
 
+## Running the benchmarks
+There are benchmarks for the htsget-search crate and for the htsget-http-actix crate. The first ones work like normal benchmarks, but the latter ones try to compare the performance of this implementation and the [reference implementation](https://github.com/ga4gh/htsget-refserver).
+There are a set of light benchmarks, and one heavy benchmark. Light benchmarks can be performed by executing:
+
+```
+cargo bench -p htsget-http-actix -- LIGHT
+```
+
+In order to run the heavy benchmark, an additional vcf file should be downloaded, and placed in the `data/vcf` directory:
+
+```
+curl ftp://ftp.1000genomes.ebi.ac.uk/vol1/ftp/data_collections/1000_genomes_project/release/20190312_biallelic_SNV_and_INDEL/ALL.chr14.shapeit2_integrated_snvindels_v2a_27022019.GRCh38.phased.vcf.gz > data/vcf/internationalgenomesample.vcf.gz
+```
+
+Then to run the heavy benchmark:
+
+```
+cargo bench -p htsget-http-actix -- HEAVY
+```
+
 ## Example Regular expressions
 In this example 'data/' is added after the first '/'.
 ```shell
-$ HTSGET_REGEX='(?P<group1>.*?)/(?P<group2>.*)' HTSGET_REPLACEMENT='$group1/data/$group2' cargo run --release -p htsget-http-actix
+$ HTSGET_REGEX='(?P<group1>.*?)/(?P<group2>.*)' HTSGET_SUBSTITUTION_STRING='$group1/data/$group2' cargo run --release -p htsget-http-actix
 ```
 For more information about the regex options look in the [documentation of the regex crate](https://docs.rs/regex/).

htsget-http-actix/benches/htsget-refserver-config.json

@@ -0,0 +1,28 @@
+{
+    "htsgetConfig": {
+        "props": {
+            "port": 8082,
+            "host": "http://localhost:8082/"
+        },
+        "reads": {
+            "dataSourceRegistry": {
+                "sources": [
+                    {
+                        "pattern": "^(?P<id>.*)$",
+                        "path": "/data/bam/{id}.bam"
+                    }
+                ]
+            }
+        },
+        "variants": {
+            "dataSourceRegistry": {
+                "sources": [
+                    {
+                        "pattern": "^(?P<id>.*)$",
+                        "path": "/data/vcf/{id}.vcf.gz"
+                    }
+                ]
+            }
+        }
+    }
+}