Feature/uni 296 scraper (#327)

* updated schema and scraper to automatically format in 18S1 or 19T2 * made routes and code to push studentVIP to database * commit to change branch * made changes to git ignore to ignore scraped data * feat: upgrade migration node to 20.4, fixed migration upsert query to remove duplicate columns * added the functions to get studentVIP reviews into the service * made routess for studentVIP revieww * bugfix: revert review/courseID return statement * started adding routes for uninotes * added backend routes for studentvip * correction: previous commit is adding routes for uniNotes * updated seed to insert uninotes reviews * added table alias for scrapped reviews to conform to postgres rules * added scrapped review schema migration and successfully seeded * course page changes (might be broken) * readded new review schema migration * fix tom's code, updated scrapped reviews schema to use 'reviews' as key in return data * added timestamp to scraped review backend schema & move its date creation to db * refactor backend to merge scraped schemas * merged studentvip and uninotes on frontend * changed sql query for get all courses to join scraped reviews table * changed all get course query to join scraped reviews table * upvotes and bookmarks * Updated the scraper python files to create sql files instead of json * fixed bookmark service and added all reviews schema in backend * fixed scraping insert statement * added scraped review id increment --------- Co-authored-by: Bob Chen <[email protected]> Co-authored-by: Jared L <[email protected]> Co-authored-by: Dylan W <[email protected]> Co-authored-by: 3syth <[email protected]> Co-authored-by: Adrian Balbalosa <[email protected]> Co-authored-by: ur-ja <[email protected]> Co-authored-by: Maximilian Falco Widjaya <[email protected]> Co-authored-by: Tom Forsyth <[email protected]> Co-authored-by: Kevin Sutandi <[email protected]> Co-authored-by: dylan <birds@DylanW-PC> Co-authored-by: Kevin Sutandi <[email protected]> Co-authored-by: Dylan <[email protected]> Co-authored-by: Maximilian Falco Widjaya <[email protected]>
devsoc-unsw · Nov 4, 2024 · 4ca2b43 · 4ca2b43
1 parent aa5ec7a
commit 4ca2b43
Show file tree

Hide file tree

Showing 25 changed files with 761 additions and 92 deletions.
diff --git a/.DS_Store b/.DS_Store
diff --git a/backend/.gitignore b/backend/.gitignore
@@ -6,4 +6,5 @@ tmp/
 temp/
 dist/
 coverage/
-.DS_Store
+.DS_Store
+data/
diff --git a/backend/package-lock.json b/backend/package-lock.json
diff --git a/backend/package.json b/backend/package.json
@@ -7,15 +7,19 @@
     "format": "prettier '**/*.ts' --write",
     "build": "tsc",
     "start": "npx prisma migrate deploy && node dist/src/index.js",
-    "dev": "NODE_ENV=dev tsx src/index.ts",
+    "dev": "NODE_ENV=dev & tsx src/index.ts",
     "dev:watch": "NODE_ENV=dev tsx watch src/index.ts",
     "test": "jest --coverage --verbose"
   },
+  "prisma": {
+    "seed": "ts-node prisma/seed.ts"
+  },
   "dependencies": {
     "@prisma/client": "^5.0.0",
     "cors": "^2.8.5",
     "envsafe": "^2.0.3",
     "express": "^4.18.2",
+    "fs": "^0.0.1-security",
     "ioredis": "^5.3.2",
     "jsonwebtoken": "^9.0.1",
     "node-fetch": "^3.3.2",
@@ -30,7 +34,7 @@
     "@types/express": "4.17.17",
     "@types/jest": "29.5.3",
     "@types/jsonwebtoken": "9.0.2",
-    "@types/node": "20.4.2",
+    "@types/node": "^20.14.10",
     "@types/swagger-ui-express": "^4.1.3",
     "@typescript-eslint/eslint-plugin": "6.0.0",
     "@typescript-eslint/parser": "6.0.0",
@@ -41,7 +45,8 @@
     "prettier": "3.2.5",
     "prisma": "^5.0.0",
     "ts-jest": "29.1.1",
+    "ts-node": "^10.9.2",
     "tsx": "^3.12.7",
-    "typescript": "^5.1.6"
+    "typescript": "^5.5.3"
   }
 }
diff --git a/...a/migrations/initial_deploy/migration.sql → ...migrations/0_initial_deploy/migration.sql b/...a/migrations/initial_deploy/migration.sql → ...migrations/0_initial_deploy/migration.sql
diff --git a/backend/prisma/migrations/20241104062037_add_reviews_scraped/migration.sql b/backend/prisma/migrations/20241104062037_add_reviews_scraped/migration.sql
@@ -0,0 +1,19 @@
+-- CreateTable
+CREATE TABLE "reviews_scraped" (
+    "review_scraped_id" UUID NOT NULL DEFAULT gen_random_uuid(),
+    "source" TEXT NOT NULL,
+    "source_id" INTEGER NOT NULL,
+    "course_code" TEXT NOT NULL,
+    "author_name" TEXT NOT NULL,
+    "title" TEXT NOT NULL,
+    "description" TEXT,
+    "term_taken" TEXT NOT NULL,
+    "created_timestamp" TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "upvotes" TEXT[],
+    "overall_rating" DOUBLE PRECISION NOT NULL,
+
+    CONSTRAINT "pk_review_scraped_id" PRIMARY KEY ("review_scraped_id")
+);
+
+-- AddForeignKey
+ALTER TABLE "reviews_scraped" ADD CONSTRAINT "fk_course_code" FOREIGN KEY ("course_code") REFERENCES "courses"("course_code") ON DELETE NO ACTION ON UPDATE NO ACTION;
diff --git a/backend/prisma/migrations/migration_lock.toml b/backend/prisma/migrations/migration_lock.toml
@@ -0,0 +1,3 @@
+# Please do not edit this file manually
+# It should be added in your version-control system (i.e. Git)
+provider = "postgresql"