From 68c8164091e9523b897f1bdab84f340245537e12 Mon Sep 17 00:00:00 2001 From: akorchyn Date: Fri, 7 Jun 2024 18:21:56 +0300 Subject: [PATCH 1/4] updated endpoints, added job to load github metadata, expanded streak table --- Cargo.lock | 1 + server/Cargo.toml | 1 + .../20240607131417_streak_repo_metadata.sql | 63 ++++++++++ server/sql/get_leaderboard.sql | 3 + server/sql/get_repo_leaderboard.sql | 12 +- server/sql/get_repos.sql | 7 ++ server/sql/get_streaks_for_user_id.sql | 12 ++ server/src/contract_pull.rs | 86 ++++++++++++++ server/src/db/mod.rs | 56 ++++++--- server/src/db/types.rs | 17 ++- server/src/entrypoints/types.rs | 53 ++++++--- server/src/github_pull.rs | 87 ++++++++++++++ server/src/lib.rs | 2 + server/src/main.rs | 112 ++++-------------- shared/src/lib.rs | 1 + shared/src/timeperiod.rs | 15 +-- 16 files changed, 397 insertions(+), 131 deletions(-) create mode 100644 server/migrations/20240607131417_streak_repo_metadata.sql create mode 100644 server/sql/get_repos.sql create mode 100644 server/sql/get_streaks_for_user_id.sql create mode 100644 server/src/contract_pull.rs create mode 100644 server/src/github_pull.rs diff --git a/Cargo.lock b/Cargo.lock index a916869..8bd6a5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3631,6 +3631,7 @@ dependencies = [ "chrono", "dotenv", "envy", + "octocrab", "reqwest 0.12.4", "rocket", "rocket_db_pools", diff --git a/server/Cargo.toml b/server/Cargo.toml index 8bc8900..a14b5cf 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -20,5 +20,6 @@ usvg = { workspace = true, features = ["text"] } rocket_prometheus.workspace = true utoipa = { workspace = true, features = ["rocket_extras", "chrono"] } utoipa-swagger-ui = { workspace = true, features = ["rocket"] } +octocrab.workspace = true shared = { workspace = true, features = ["client"] } diff --git a/server/migrations/20240607131417_streak_repo_metadata.sql b/server/migrations/20240607131417_streak_repo_metadata.sql new file mode 100644 index 0000000..1a29c3c --- /dev/null +++ b/server/migrations/20240607131417_streak_repo_metadata.sql @@ -0,0 +1,63 @@ +BEGIN; + +-- Step 1: Create the new streak table +CREATE TABLE IF NOT EXISTS streak ( + id INTEGER PRIMARY KEY, + name TEXT UNIQUE NOT NULL, + period TEXT NOT NULL +); + +-- Step 2: Insert any missing streak entries into the new streak table +INSERT INTO + streak (id, name, period) +VALUES + (0, 'Weekly Pull Request', 'Weekly'), + ( + 1, + 'Monthly Pull Request with score higher 8', + 'Monthly' + ); + +ALTER TABLE + streak_user_data +ADD + COLUMN new_streak_id INTEGER; + +-- Step 4: Copy data from the old streak_id column to the new_streak_id column +UPDATE + streak_user_data +SET + new_streak_id = streak_user_data.streak_id; + +-- Step 5: Drop the old streak_id column +ALTER TABLE + streak_user_data DROP COLUMN streak_id; + +-- Step 6: Rename the new_streak_id column to streak_id +ALTER TABLE + streak_user_data RENAME COLUMN new_streak_id TO streak_id; + +-- Step 7: Add the foreign key constraint to the new streak_id column +ALTER TABLE + streak_user_data +ADD + CONSTRAINT fk_streak_id FOREIGN KEY (streak_id) REFERENCES streak(id) ON DELETE CASCADE; + +-- Step 8: Re-add the primary key constraint +ALTER TABLE + streak_user_data +ADD + PRIMARY KEY (user_id, streak_id); + +ALTER TABLE + repos +ADD + COLUMN primary_language TEXT, +ADD + COLUMN open_issues INTEGER, +ADD + COLUMN stars INTEGER, +ADD + COLUMN forks INTEGER; + +COMMIT; diff --git a/server/sql/get_leaderboard.sql b/server/sql/get_leaderboard.sql index 1d6e80e..50a4725 100644 --- a/server/sql/get_leaderboard.sql +++ b/server/sql/get_leaderboard.sql @@ -8,11 +8,14 @@ SELECT prs_merged, best as streak_best, amount as streak_amount, + period as streak_type, + streak.name as streak_name, latest_time_string as streak_latest_time_string FROM user_period_data JOIN users ON users.id = user_period_data.user_id JOIN streak_user_data ON streak_user_data.user_id = users.id + JOIN streak ON streak.id = streak_user_data.streak_id WHERE period_type = $1 and streak_user_data.streak_id = $2 diff --git a/server/sql/get_repo_leaderboard.sql b/server/sql/get_repo_leaderboard.sql index 14888d1..3649779 100644 --- a/server/sql/get_repo_leaderboard.sql +++ b/server/sql/get_repo_leaderboard.sql @@ -20,7 +20,11 @@ SELECT r.name AS name, COALESCE(COUNT(pr.id), 0) AS total_prs, COALESCE(SUM(pr.score), 0) AS total_score, - tc.contributor_name AS top_contributor + tc.contributor_name AS top_contributor, + r.primary_language, + r.open_issues, + r.stars, + r.forks FROM repos r JOIN organizations o ON r.organization_id = o.id @@ -30,7 +34,11 @@ FROM GROUP BY o.name, r.name, - tc.contributor_name + tc.contributor_name, + r.primary_language, + r.open_issues, + r.stars, + r.forks ORDER BY total_score DESC, total_prs DESC diff --git a/server/sql/get_repos.sql b/server/sql/get_repos.sql new file mode 100644 index 0000000..cce8f67 --- /dev/null +++ b/server/sql/get_repos.sql @@ -0,0 +1,7 @@ +select + r.id as repo_id, + r.name as repo, + o.name as organization +from + repos as r + JOIN organizations o ON r.organization_id = o.id diff --git a/server/sql/get_streaks_for_user_id.sql b/server/sql/get_streaks_for_user_id.sql new file mode 100644 index 0000000..e3e2301 --- /dev/null +++ b/server/sql/get_streaks_for_user_id.sql @@ -0,0 +1,12 @@ +SELECT + streak.id as streak_id, + name, + period as streak_type, + amount, + best, + latest_time_string +FROM + streak_user_data + JOIN streak ON streak.id = streak_user_data.streak_id +WHERE + user_id = $1 diff --git a/server/src/contract_pull.rs b/server/src/contract_pull.rs new file mode 100644 index 0000000..f47f4de --- /dev/null +++ b/server/src/contract_pull.rs @@ -0,0 +1,86 @@ +use std::{ + sync::{atomic::AtomicBool, Arc}, + time::Duration, +}; + +use chrono::DateTime; +use rocket::fairing::AdHoc; +use rocket_db_pools::Database; +use shared::{near::NearClient, TimePeriod}; + +use crate::db::DB; + +async fn fetch_and_store_users(near_client: &NearClient, db: &DB) -> anyhow::Result<()> { + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_nanos(); + let periods = [TimePeriod::Month, TimePeriod::Quarter, TimePeriod::AllTime] + .into_iter() + .map(|e| e.time_string(timestamp as u64)) + .collect(); + let users = near_client.users(periods).await?; + for user in users { + let user_id = db.upsert_user(&user.name).await?; + for (period, data) in user.period_data { + db.upsert_user_period_data(period, &data, user_id).await?; + } + for (streak_id, streak_data) in user.streaks { + db.upsert_streak_user_data(&streak_data, streak_id as i32, user_id) + .await?; + } + } + + Ok(()) +} + +async fn fetch_and_store_prs(near_client: &NearClient, db: &DB) -> anyhow::Result<()> { + let prs = near_client.prs().await?; + for (pr, executed) in prs { + let organization_id = db.upsert_organization(&pr.organization).await?; + let repo_id = db.upsert_repo(organization_id, &pr.repo).await?; + let author_id = db.upsert_user(&pr.author).await?; + let _ = db + .upsert_pull_request( + repo_id, + pr.number as i32, + author_id, + DateTime::from_timestamp_nanos(pr.created_at as i64).naive_utc(), + pr.merged_at + .map(|t| DateTime::from_timestamp_nanos(t as i64).naive_utc()), + pr.score(), + executed, + ) + .await?; + } + Ok(()) +} + +async fn fetch_and_store_all_data(near_client: &NearClient, db: &DB) -> anyhow::Result<()> { + fetch_and_store_users(near_client, db).await?; + fetch_and_store_prs(near_client, db).await?; + Ok(()) +} + +pub fn stage(client: NearClient, sleep_duration: Duration, atomic_bool: Arc) -> AdHoc { + rocket::fairing::AdHoc::on_liftoff("Load users from Near every X minutes", move |rocket| { + Box::pin(async move { + // Get an actual DB connection + let db = DB::fetch(rocket) + .expect("Failed to get DB connection") + .clone(); + + rocket::tokio::spawn(async move { + let mut interval = rocket::tokio::time::interval(sleep_duration); + let near_client = client; + while atomic_bool.load(std::sync::atomic::Ordering::Relaxed) { + interval.tick().await; + + // Execute a query of some kind + if let Err(e) = fetch_and_store_all_data(&near_client, &db).await { + rocket::error!("Failed to fetch and store data: {:#?}", e); + } + } + }); + }) + }) +} diff --git a/server/src/db/mod.rs b/server/src/db/mod.rs index f14412e..f4b9123 100644 --- a/server/src/db/mod.rs +++ b/server/src/db/mod.rs @@ -14,7 +14,10 @@ pub mod types; use types::LeaderboardRecord; -use self::types::{RepoRecord, StreakRecord, UserContributionRecord, UserPeriodRecord, UserRecord}; +use self::types::{ + RepoLeaderboardRecord, RepoRecord, StreakRecord, UserContributionRecord, UserPeriodRecord, + UserRecord, +}; impl DB { pub async fn upsert_user(&self, user: &str) -> anyhow::Result { @@ -153,6 +156,30 @@ impl DB { Ok(()) } + pub async fn update_repo_metadata( + &self, + repo_id: i32, + stars: u32, + forks: u32, + open_issues: u32, + primary_language: Option, + ) -> anyhow::Result<()> { + sqlx::query!( + r#" + UPDATE repos + SET stars = $1, forks = $2, open_issues = $3, primary_language = $4 + WHERE id = $5"#, + stars as i32, + forks as i32, + open_issues as i32, + primary_language, + repo_id + ) + .execute(&self.0) + .await?; + Ok(()) + } + pub async fn get_user( &self, name: &str, @@ -178,17 +205,10 @@ impl DB { .fetch_all(&self.0) .await?; - let streak_recs: Vec = sqlx::query_as!( - StreakRecord, - r#" - SELECT streak_id, amount, best, latest_time_string - FROM streak_user_data - WHERE user_id = $1 - "#, - user_rec - ) - .fetch_all(&self.0) - .await?; + let streak_recs: Vec = + sqlx::query_file_as!(StreakRecord, "./sql/get_streaks_for_user_id.sql", user_rec) + .fetch_all(&self.0) + .await?; let mut leaderboard_places = Vec::with_capacity(place_strings.len()); for place in place_strings { @@ -254,12 +274,12 @@ impl DB { &self, page: i64, limit: i64, - ) -> anyhow::Result<(Vec, u64)> { + ) -> anyhow::Result<(Vec, u64)> { let offset = page * limit; // COALESCE is used to return 0 if there are no PRs for a repo // But sqlx still thinks that it's NONE let records = sqlx::query_file_as_unchecked!( - RepoRecord, + RepoLeaderboardRecord, "./sql/get_repo_leaderboard.sql", limit, offset @@ -339,6 +359,14 @@ impl DB { .map(|r| (r.name, r.total_score.unwrap_or_default())) .collect()) } + + pub async fn get_repos(&self) -> anyhow::Result> { + let rec = sqlx::query_file_as!(RepoRecord, "./sql/get_repos.sql") + .fetch_all(&self.0) + .await?; + + Ok(rec) + } } async fn run_migrations(rocket: Rocket) -> fairing::Result { diff --git a/server/src/db/types.rs b/server/src/db/types.rs index 3b1eab0..f46be95 100644 --- a/server/src/db/types.rs +++ b/server/src/db/types.rs @@ -11,6 +11,8 @@ pub struct LeaderboardRecord { pub prs_merged: i32, pub streak_best: i32, pub streak_amount: i32, + pub streak_name: String, + pub streak_type: String, pub streak_latest_time_string: String, } @@ -27,6 +29,8 @@ pub struct UserPeriodRecord { #[derive(Debug, Clone, sqlx::FromRow, Serialize, Deserialize, Default)] pub struct StreakRecord { pub streak_id: i32, + pub name: String, + pub streak_type: String, pub amount: i32, pub best: i32, pub latest_time_string: String, @@ -52,12 +56,16 @@ impl UserRecord { } #[derive(Debug, Clone, sqlx::FromRow, Serialize, Deserialize)] -pub struct RepoRecord { +pub struct RepoLeaderboardRecord { pub organization: String, pub name: String, pub total_prs: i64, pub total_score: i64, pub top_contributor: GithubHandle, + pub stars: i32, + pub open_issues: i32, + pub primary_language: Option, + pub forks: i32, } #[derive(Debug, Clone, sqlx::FromRow, Serialize, Deserialize)] @@ -71,3 +79,10 @@ pub struct UserContributionRecord { pub created_at: chrono::NaiveDateTime, pub merged_at: Option, } + +#[derive(Debug, Clone, sqlx::FromRow, Serialize, Deserialize)] +pub struct RepoRecord { + pub organization: String, + pub repo: String, + pub repo_id: i32, +} diff --git a/server/src/entrypoints/types.rs b/server/src/entrypoints/types.rs index d849ce0..78b2655 100644 --- a/server/src/entrypoints/types.rs +++ b/server/src/entrypoints/types.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use chrono::NaiveDateTime; use race_of_sloths_server::db::types::{ - LeaderboardRecord, RepoRecord, UserContributionRecord, UserRecord, + LeaderboardRecord, RepoLeaderboardRecord, UserContributionRecord, UserRecord, }; use serde::{Deserialize, Serialize}; use shared::TimePeriod; @@ -49,24 +49,25 @@ impl GithubMeta { pub struct RepoResponse { pub name: String, pub organization: GithubMeta, - pub project_language: String, - pub project_language_color: String, - pub contributor_of_the_month: String, + pub repo_language: Option, + pub stars: u32, + pub forks: u32, pub open_issues: u32, + pub contributor_of_the_month: String, pub contributions_with_sloth: u32, pub total_score: u32, } -impl From for RepoResponse { - fn from(record: RepoRecord) -> Self { +impl From for RepoResponse { + fn from(record: RepoLeaderboardRecord) -> Self { Self { name: record.name, organization: GithubMeta::new(record.organization), - // TODO: fix these fields - project_language: "RUST".to_string(), - project_language_color: "#000000".to_string(), + repo_language: record.primary_language, + stars: record.stars as u32, + forks: record.forks as u32, + open_issues: record.open_issues as u32, contributor_of_the_month: record.top_contributor, - open_issues: 0, contributions_with_sloth: record.total_prs as u32, total_score: record.total_score as u32, } @@ -91,9 +92,11 @@ impl From for LeaderboardResponse { rating: 0, contributions: record.prs_opened as u32, streak: Streak::new( + record.streak_name, record.streak_amount as u32, record.streak_best as u32, - &record.streak_latest_time_string, + record.streak_latest_time_string, + record.streak_type, ), merged_prs: record.prs_merged as u32, score: record.total_score as u32, @@ -103,13 +106,21 @@ impl From for LeaderboardResponse { #[derive(Clone, Debug, Serialize, Deserialize, Default, ToSchema)] pub struct Streak { + name: String, + streak_type: String, current: u32, longest: u32, } impl Streak { - pub fn new(current: u32, longest: u32, time_period_string: &str) -> Self { - if let Some(time_period) = TimePeriod::from_time_period_string(time_period_string) { + pub fn new( + name: String, + current: u32, + longest: u32, + time_period_string: String, + streak_type: String, + ) -> Self { + if let Some(time_period) = TimePeriod::from_streak_type(&streak_type) { let current_time = chrono::Utc::now().timestamp_nanos_opt().unwrap_or_default(); let previous_period = time_period .previous_period(current_time as u64) @@ -119,11 +130,18 @@ impl Streak { if current_time_string == time_period_string || previous_period_string == time_period_string { - return Self { current, longest }; + return Self { + name, + streak_type, + current, + longest, + }; }; } Self { + name, + streak_type, current: 0, longest, } @@ -157,12 +175,13 @@ impl From for UserProfile { .into_iter() .map(|streak| { ( - // TODO: We should write here a string name - streak.streak_id.to_string(), + streak.name.clone(), Streak::new( + streak.name, streak.amount as u32, streak.best as u32, - &streak.latest_time_string, + streak.latest_time_string, + streak.streak_type, ), ) }) diff --git a/server/src/github_pull.rs b/server/src/github_pull.rs new file mode 100644 index 0000000..b3c7f46 --- /dev/null +++ b/server/src/github_pull.rs @@ -0,0 +1,87 @@ +use std::{ + sync::{atomic::AtomicBool, Arc}, + time::Duration, +}; + +use octocrab::Octocrab; +use rocket::fairing::AdHoc; +use rocket_db_pools::Database; + +use crate::db::DB; + +struct RepoMetadata { + stars: u32, + forks: u32, + open_issues: u32, + primary_language: Option, +} + +pub struct GithubClient { + octocrab: Octocrab, +} + +impl GithubClient { + pub fn new(github_token: String) -> anyhow::Result { + let octocrab = octocrab::Octocrab::builder() + .personal_token(github_token) + .build()?; + Ok(Self { octocrab }) + } + + async fn repo_metadata(&self, org: &str, repo: &str) -> anyhow::Result { + let repo = self.octocrab.repos(org, repo).get().await?; + Ok(RepoMetadata { + stars: repo.stargazers_count.unwrap_or_default(), + forks: repo.forks_count.unwrap_or_default(), + open_issues: repo.open_issues_count.unwrap_or_default(), + primary_language: repo.language.map(|l| l.to_string()), + }) + } +} + +async fn fetch_repos_metadata(github: &GithubClient, db: &DB) -> anyhow::Result<()> { + let repos = db.get_repos().await?; + for repo in repos { + let metadata = github.repo_metadata(&repo.organization, &repo.repo).await?; + db.update_repo_metadata( + repo.repo_id, + metadata.stars, + metadata.forks, + metadata.open_issues, + metadata.primary_language, + ) + .await?; + } + Ok(()) +} + +pub fn stage( + github_client: GithubClient, + sleep_duration: Duration, + atomic_bool: Arc, +) -> AdHoc { + rocket::fairing::AdHoc::on_liftoff( + "Loads github repository metadata every X minutes", + move |rocket| { + Box::pin(async move { + // Get an actual DB connection + let db = DB::fetch(rocket) + .expect("Failed to get DB connection") + .clone(); + + rocket::tokio::spawn(async move { + let mut interval = rocket::tokio::time::interval(sleep_duration); + let github_client = github_client; + while atomic_bool.load(std::sync::atomic::Ordering::Relaxed) { + interval.tick().await; + + // Execute a query of some kind + if let Err(e) = fetch_repos_metadata(&github_client, &db).await { + rocket::error!("Failed to fetch and store github data: {:#?}", e); + } + } + }); + }) + }, + ) +} diff --git a/server/src/lib.rs b/server/src/lib.rs index 788339c..9d6dd34 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -1,2 +1,4 @@ +pub mod contract_pull; pub mod db; +pub mod github_pull; pub mod svg; diff --git a/server/src/main.rs b/server/src/main.rs index f8dde2d..0ff8381 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -6,13 +6,10 @@ mod entrypoints; use std::sync::Arc; use std::time::Duration; -use chrono::DateTime; use entrypoints::ApiDoc; -use rocket_db_pools::Database; use shared::near::NearClient; -use shared::TimePeriod; -use race_of_sloths_server::db::{self, DB}; +use race_of_sloths_server::{contract_pull, db, github_pull}; use utoipa::OpenApi; use utoipa_swagger_ui::SwaggerUi; @@ -21,7 +18,9 @@ pub struct Env { contract: String, secret_key: String, is_mainnet: bool, - sleep_duration_in_minutes: Option, + near_timeout_in_minutes: Option, + github_timeout_in_minutes: Option, + github_token: String, } #[launch] @@ -29,49 +28,33 @@ async fn rocket() -> _ { dotenv::dotenv().ok(); let env = envy::from_env::().expect("Failed to load environment variables"); - let sleep_duration = - Duration::from_secs(env.sleep_duration_in_minutes.unwrap_or(10) as u64 * 60); + let near_sleep = Duration::from_secs(env.near_timeout_in_minutes.unwrap_or(10) as u64 * 60); + let github_sleep = Duration::from_secs(env.github_timeout_in_minutes.unwrap_or(60) as u64 * 60); let atomic_bool = Arc::new(std::sync::atomic::AtomicBool::new(true)); - let atomic_bool_clone = atomic_bool.clone(); let prometheus = rocket_prometheus::PrometheusMetrics::new(); + let near_client = NearClient::new(env.contract.clone(), env.secret_key.clone(), env.is_mainnet) + .await + .expect("Failed to create Near client"); + rocket::build() .attach(db::stage()) - .attach(rocket::fairing::AdHoc::on_liftoff( - "Load users from Near every X minutes", - move |rocket| { - Box::pin(async move { - // Get an actual DB connection - let db = DB::fetch(rocket) - .expect("Failed to get DB connection") - .clone(); - - rocket::tokio::spawn(async move { - let mut interval = rocket::tokio::time::interval(sleep_duration); - let near_client = NearClient::new( - env.contract.clone(), - env.secret_key.clone(), - env.is_mainnet, - ) - .await - .expect("Failed to create Near client"); - while atomic_bool.load(std::sync::atomic::Ordering::Relaxed) { - interval.tick().await; - - // Execute a query of some kind - if let Err(e) = fetch_and_store_all_data(&near_client, &db).await { - rocket::error!("Failed to fetch and store data: {:#?}", e); - } - } - }); - }) - }, + .attach(contract_pull::stage( + near_client, + near_sleep, + atomic_bool.clone(), + )) + .attach(github_pull::stage( + github_pull::GithubClient::new(env.github_token.clone()) + .expect("Failed to create Github client"), + github_sleep, + atomic_bool.clone(), )) .attach(rocket::fairing::AdHoc::on_shutdown( - "Stop loading users from Near", + "Stop loading users from Near and Github metadata", |_| { Box::pin(async move { - atomic_bool_clone.store(false, std::sync::atomic::Ordering::Relaxed); + atomic_bool.store(false, std::sync::atomic::Ordering::Relaxed); }) }, )) @@ -83,54 +66,3 @@ async fn rocket() -> _ { .attach(entrypoints::stage()) .mount("/metrics", prometheus) } - -async fn fetch_and_store_users(near_client: &NearClient, db: &DB) -> anyhow::Result<()> { - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH)? - .as_nanos(); - let periods = [TimePeriod::Month, TimePeriod::Quarter, TimePeriod::AllTime] - .into_iter() - .map(|e| e.time_string(timestamp as u64)) - .collect(); - let users = near_client.users(periods).await?; - for user in users { - let user_id = db.upsert_user(&user.name).await?; - for (period, data) in user.period_data { - db.upsert_user_period_data(period, &data, user_id).await?; - } - for (streak_id, streak_data) in user.streaks { - db.upsert_streak_user_data(&streak_data, streak_id as i32, user_id) - .await?; - } - } - - Ok(()) -} - -async fn fetch_and_store_prs(near_client: &NearClient, db: &DB) -> anyhow::Result<()> { - let prs = near_client.prs().await?; - for (pr, executed) in prs { - let organization_id = db.upsert_organization(&pr.organization).await?; - let repo_id = db.upsert_repo(organization_id, &pr.repo).await?; - let author_id = db.upsert_user(&pr.author).await?; - let _ = db - .upsert_pull_request( - repo_id, - pr.number as i32, - author_id, - DateTime::from_timestamp_nanos(pr.created_at as i64).naive_utc(), - pr.merged_at - .map(|t| DateTime::from_timestamp_nanos(t as i64).naive_utc()), - pr.score(), - executed, - ) - .await?; - } - Ok(()) -} - -async fn fetch_and_store_all_data(near_client: &NearClient, db: &DB) -> anyhow::Result<()> { - fetch_and_store_users(near_client, db).await?; - fetch_and_store_prs(near_client, db).await?; - Ok(()) -} diff --git a/shared/src/lib.rs b/shared/src/lib.rs index 1480e70..9b8f22c 100644 --- a/shared/src/lib.rs +++ b/shared/src/lib.rs @@ -10,6 +10,7 @@ mod timeperiod; #[cfg(feature = "github")] pub mod github; + #[cfg(feature = "client")] pub mod near; diff --git a/shared/src/timeperiod.rs b/shared/src/timeperiod.rs index e75aa1b..2473f04 100644 --- a/shared/src/timeperiod.rs +++ b/shared/src/timeperiod.rs @@ -32,13 +32,14 @@ pub enum TimePeriod { } impl TimePeriod { - pub fn from_time_period_string(period: &str) -> Option { - match period { - a if a.contains('W') => Some(TimePeriod::Week), - a if a.contains('Q') => Some(TimePeriod::Quarter), - a if a.len() == 8 => Some(TimePeriod::Day), - a if a.len() == 6 => Some(TimePeriod::Month), - "all-time" => Some(TimePeriod::AllTime), + pub fn from_streak_type(type_: &str) -> Option { + match type_.to_lowercase().as_str() { + "daily" => Some(Self::Day), + "weekly" => Some(Self::Week), + "monthly" => Some(Self::Month), + "quarterly" => Some(Self::Quarter), + "yearly" => Some(Self::Year), + "all-time" => Some(Self::AllTime), _ => None, } } From 32eb1e2aecc2ced4a374e20785f40d2456a6f46b Mon Sep 17 00:00:00 2001 From: akorchyn Date: Fri, 7 Jun 2024 18:23:04 +0300 Subject: [PATCH 2/4] sqlx data for compilation --- ...67b8e32f2b5bd59ced2671fbc3a4f6ff0a52e.json | 32 +++++++++++++++++++ ...625b5e5ed4d93130528b08d25db173285465d.json | 18 +++++++++++ ...467876e6d618eaea806d6729dbbe149ce972.json} | 16 ++++++++-- ...ca0d185f41747644882d6f3a6776a570d921.json} | 30 +++++++++++++++-- ...e11b55967c3bab50301ab41f89230d4e8e50.json} | 20 +++++++++--- 5 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 server/.sqlx/query-30a16d6f56b3c3f31ae89171e6767b8e32f2b5bd59ced2671fbc3a4f6ff0a52e.json create mode 100644 server/.sqlx/query-3a366412f033f50731f39268478625b5e5ed4d93130528b08d25db173285465d.json rename server/.sqlx/{query-2c2f326af19143665adf083920b4b06312034c42a3a2fae869cd0d0a4bffcedd.json => query-418d7211bb96b937b302fe84005f467876e6d618eaea806d6729dbbe149ce972.json} (66%) rename server/.sqlx/{query-8983a89ef78684004600907c856b370305af1de54aac665d6e36d60b6a36a5f3.json => query-4215feff79a7401c591abe377138ca0d185f41747644882d6f3a6776a570d921.json} (54%) rename server/.sqlx/{query-c948b8c82bac8f0787c05d102c5e2119d8ec12683ef55fb70c9e0f1100560b8c.json => query-8ea1b590ec23cf48854d034b65cfe11b55967c3bab50301ab41f89230d4e8e50.json} (52%) diff --git a/server/.sqlx/query-30a16d6f56b3c3f31ae89171e6767b8e32f2b5bd59ced2671fbc3a4f6ff0a52e.json b/server/.sqlx/query-30a16d6f56b3c3f31ae89171e6767b8e32f2b5bd59ced2671fbc3a4f6ff0a52e.json new file mode 100644 index 0000000..d3b65bb --- /dev/null +++ b/server/.sqlx/query-30a16d6f56b3c3f31ae89171e6767b8e32f2b5bd59ced2671fbc3a4f6ff0a52e.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "select\n r.id as repo_id,\n r.name as repo,\n o.name as organization\nfrom\n repos as r\n JOIN organizations o ON r.organization_id = o.id\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "repo_id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "repo", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "organization", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "30a16d6f56b3c3f31ae89171e6767b8e32f2b5bd59ced2671fbc3a4f6ff0a52e" +} diff --git a/server/.sqlx/query-3a366412f033f50731f39268478625b5e5ed4d93130528b08d25db173285465d.json b/server/.sqlx/query-3a366412f033f50731f39268478625b5e5ed4d93130528b08d25db173285465d.json new file mode 100644 index 0000000..0d3d907 --- /dev/null +++ b/server/.sqlx/query-3a366412f033f50731f39268478625b5e5ed4d93130528b08d25db173285465d.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repos\n SET stars = $1, forks = $2, open_issues = $3, primary_language = $4\n WHERE id = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4", + "Int4", + "Text", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "3a366412f033f50731f39268478625b5e5ed4d93130528b08d25db173285465d" +} diff --git a/server/.sqlx/query-2c2f326af19143665adf083920b4b06312034c42a3a2fae869cd0d0a4bffcedd.json b/server/.sqlx/query-418d7211bb96b937b302fe84005f467876e6d618eaea806d6729dbbe149ce972.json similarity index 66% rename from server/.sqlx/query-2c2f326af19143665adf083920b4b06312034c42a3a2fae869cd0d0a4bffcedd.json rename to server/.sqlx/query-418d7211bb96b937b302fe84005f467876e6d618eaea806d6729dbbe149ce972.json index ef799d4..b620304 100644 --- a/server/.sqlx/query-2c2f326af19143665adf083920b4b06312034c42a3a2fae869cd0d0a4bffcedd.json +++ b/server/.sqlx/query-418d7211bb96b937b302fe84005f467876e6d618eaea806d6729dbbe149ce972.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n users.name,\n period_type,\n total_score,\n executed_prs,\n largest_score,\n prs_opened,\n prs_merged,\n best as streak_best,\n amount as streak_amount,\n latest_time_string as streak_latest_time_string\nFROM\n user_period_data\n JOIN users ON users.id = user_period_data.user_id\n JOIN streak_user_data ON streak_user_data.user_id = users.id\nWHERE\n period_type = $1\n and streak_user_data.streak_id = $2\nORDER BY\n total_score DESC\nLIMIT\n $3 OFFSET $4\n", + "query": "SELECT\n users.name,\n period_type,\n total_score,\n executed_prs,\n largest_score,\n prs_opened,\n prs_merged,\n best as streak_best,\n amount as streak_amount,\n period as streak_type,\n streak.name as streak_name,\n latest_time_string as streak_latest_time_string\nFROM\n user_period_data\n JOIN users ON users.id = user_period_data.user_id\n JOIN streak_user_data ON streak_user_data.user_id = users.id\n JOIN streak ON streak.id = streak_user_data.streak_id\nWHERE\n period_type = $1\n and streak_user_data.streak_id = $2\nORDER BY\n total_score DESC\nLIMIT\n $3 OFFSET $4\n", "describe": { "columns": [ { @@ -50,6 +50,16 @@ }, { "ordinal": 9, + "name": "streak_type", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "streak_name", + "type_info": "Text" + }, + { + "ordinal": 11, "name": "streak_latest_time_string", "type_info": "Text" } @@ -72,8 +82,10 @@ false, false, false, + false, + false, false ] }, - "hash": "2c2f326af19143665adf083920b4b06312034c42a3a2fae869cd0d0a4bffcedd" + "hash": "418d7211bb96b937b302fe84005f467876e6d618eaea806d6729dbbe149ce972" } diff --git a/server/.sqlx/query-8983a89ef78684004600907c856b370305af1de54aac665d6e36d60b6a36a5f3.json b/server/.sqlx/query-4215feff79a7401c591abe377138ca0d185f41747644882d6f3a6776a570d921.json similarity index 54% rename from server/.sqlx/query-8983a89ef78684004600907c856b370305af1de54aac665d6e36d60b6a36a5f3.json rename to server/.sqlx/query-4215feff79a7401c591abe377138ca0d185f41747644882d6f3a6776a570d921.json index 0189af4..efd8a31 100644 --- a/server/.sqlx/query-8983a89ef78684004600907c856b370305af1de54aac665d6e36d60b6a36a5f3.json +++ b/server/.sqlx/query-4215feff79a7401c591abe377138ca0d185f41747644882d6f3a6776a570d921.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH top_contributors AS (\n SELECT\n pr.repo_id,\n u.name AS contributor_name,\n SUM(pr.score) AS total_score,\n ROW_NUMBER() OVER (\n PARTITION BY pr.repo_id\n ORDER BY\n SUM(pr.score) DESC\n ) AS rank\n FROM\n pull_requests pr\n JOIN users u ON pr.author_id = u.id\n GROUP BY\n pr.repo_id,\n u.id\n)\nSELECT\n o.name AS organization,\n r.name AS name,\n COALESCE(COUNT(pr.id), 0) AS total_prs,\n COALESCE(SUM(pr.score), 0) AS total_score,\n tc.contributor_name AS top_contributor\nFROM\n repos r\n JOIN organizations o ON r.organization_id = o.id\n LEFT JOIN pull_requests pr ON pr.repo_id = r.id\n LEFT JOIN top_contributors tc ON tc.repo_id = r.id\n AND tc.rank = 1\nGROUP BY\n o.name,\n r.name,\n tc.contributor_name\nORDER BY\n total_score DESC,\n total_prs DESC\nLIMIT\n $1 OFFSET $2;\n", + "query": "WITH top_contributors AS (\n SELECT\n pr.repo_id,\n u.name AS contributor_name,\n SUM(pr.score) AS total_score,\n ROW_NUMBER() OVER (\n PARTITION BY pr.repo_id\n ORDER BY\n SUM(pr.score) DESC\n ) AS rank\n FROM\n pull_requests pr\n JOIN users u ON pr.author_id = u.id\n GROUP BY\n pr.repo_id,\n u.id\n)\nSELECT\n o.name AS organization,\n r.name AS name,\n COALESCE(COUNT(pr.id), 0) AS total_prs,\n COALESCE(SUM(pr.score), 0) AS total_score,\n tc.contributor_name AS top_contributor,\n r.primary_language,\n r.open_issues,\n r.stars,\n r.forks\nFROM\n repos r\n JOIN organizations o ON r.organization_id = o.id\n LEFT JOIN pull_requests pr ON pr.repo_id = r.id\n LEFT JOIN top_contributors tc ON tc.repo_id = r.id\n AND tc.rank = 1\nGROUP BY\n o.name,\n r.name,\n tc.contributor_name,\n r.primary_language,\n r.open_issues,\n r.stars,\n r.forks\nORDER BY\n total_score DESC,\n total_prs DESC\nLIMIT\n $1 OFFSET $2;\n", "describe": { "columns": [ { @@ -27,6 +27,26 @@ "ordinal": 4, "name": "top_contributor", "type_info": "Text" + }, + { + "ordinal": 5, + "name": "primary_language", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "open_issues", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "stars", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "forks", + "type_info": "Int4" } ], "parameters": { @@ -40,8 +60,12 @@ false, null, null, - false + false, + true, + true, + true, + true ] }, - "hash": "8983a89ef78684004600907c856b370305af1de54aac665d6e36d60b6a36a5f3" + "hash": "4215feff79a7401c591abe377138ca0d185f41747644882d6f3a6776a570d921" } diff --git a/server/.sqlx/query-c948b8c82bac8f0787c05d102c5e2119d8ec12683ef55fb70c9e0f1100560b8c.json b/server/.sqlx/query-8ea1b590ec23cf48854d034b65cfe11b55967c3bab50301ab41f89230d4e8e50.json similarity index 52% rename from server/.sqlx/query-c948b8c82bac8f0787c05d102c5e2119d8ec12683ef55fb70c9e0f1100560b8c.json rename to server/.sqlx/query-8ea1b590ec23cf48854d034b65cfe11b55967c3bab50301ab41f89230d4e8e50.json index d4e884d..d4bc73b 100644 --- a/server/.sqlx/query-c948b8c82bac8f0787c05d102c5e2119d8ec12683ef55fb70c9e0f1100560b8c.json +++ b/server/.sqlx/query-8ea1b590ec23cf48854d034b65cfe11b55967c3bab50301ab41f89230d4e8e50.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT streak_id, amount, best, latest_time_string\n FROM streak_user_data\n WHERE user_id = $1\n ", + "query": "SELECT\n streak.id as streak_id,\n name,\n period as streak_type,\n amount,\n best,\n latest_time_string\nFROM\n streak_user_data\n JOIN streak ON streak.id = streak_user_data.streak_id\nWHERE\n user_id = $1\n", "describe": { "columns": [ { @@ -10,16 +10,26 @@ }, { "ordinal": 1, + "name": "name", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "streak_type", + "type_info": "Text" + }, + { + "ordinal": 3, "name": "amount", "type_info": "Int4" }, { - "ordinal": 2, + "ordinal": 4, "name": "best", "type_info": "Int4" }, { - "ordinal": 3, + "ordinal": 5, "name": "latest_time_string", "type_info": "Text" } @@ -30,11 +40,13 @@ ] }, "nullable": [ + false, + false, false, false, false, false ] }, - "hash": "c948b8c82bac8f0787c05d102c5e2119d8ec12683ef55fb70c9e0f1100560b8c" + "hash": "8ea1b590ec23cf48854d034b65cfe11b55967c3bab50301ab41f89230d4e8e50" } From ba72f93f79222e96f248157afc8e41da54149114 Mon Sep 17 00:00:00 2001 From: akorchyn Date: Fri, 7 Jun 2024 18:27:58 +0300 Subject: [PATCH 3/4] small language fix --- server/src/github_pull.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/github_pull.rs b/server/src/github_pull.rs index b3c7f46..9017b16 100644 --- a/server/src/github_pull.rs +++ b/server/src/github_pull.rs @@ -34,7 +34,9 @@ impl GithubClient { stars: repo.stargazers_count.unwrap_or_default(), forks: repo.forks_count.unwrap_or_default(), open_issues: repo.open_issues_count.unwrap_or_default(), - primary_language: repo.language.map(|l| l.to_string()), + primary_language: repo + .language + .and_then(|l| l.as_str().map(ToString::to_string)), }) } } From ab25f24eef27327eed4ddaafe820f80e2784c2f1 Mon Sep 17 00:00:00 2001 From: akorchyn Date: Fri, 7 Jun 2024 19:03:12 +0300 Subject: [PATCH 4/4] log todo --- server/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main.rs b/server/src/main.rs index 0ff8381..c493de0 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -36,6 +36,7 @@ async fn rocket() -> _ { let near_client = NearClient::new(env.contract.clone(), env.secret_key.clone(), env.is_mainnet) .await .expect("Failed to create Near client"); + // TODO: after 0.6.0 release, we should use tracing for redirecting warns and errors to the telegram rocket::build() .attach(db::stage())