From bc5b734cfcb098958a411391842e9120d8052129 Mon Sep 17 00:00:00 2001 From: Henry Post Date: Mon, 2 Dec 2019 15:44:15 -0600 Subject: [PATCH 01/42] add example url for reddit --- bowser/bowserHTTPAPI.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bowser/bowserHTTPAPI.py b/bowser/bowserHTTPAPI.py index 491fa3f..d9ab668 100644 --- a/bowser/bowserHTTPAPI.py +++ b/bowser/bowserHTTPAPI.py @@ -64,7 +64,9 @@ def index(): "route-url": url_for("routes"), "example-urls": [ (url_for('generate_csv') + "?boards=x,pol&flaggers=NSA_PRISM,TERRORISM&start_page=3&stop_page=10"), - (url_for('generate_json') + "?boards=pol,s4s,x&flaggers=NSA_ECHELON,RACISM&start_page=1&stop_page=3") + (url_for( + 'generate_4plebs_json') + "?boards=pol,s4s,x&flaggers=NSA_ECHELON,RACISM&start_page=1&stop_page=3"), + (url_for('generate_reddit_json') + '?subreddit=Sino'), ] }) @@ -196,7 +198,7 @@ def generate_reddit_json(): 'thread_list': threadlist}) -def _generate_csv_string(boards: str, flaggers: str, start_page: str, stop_page: str) -> str: +def _generate_csv_string_4plebs(boards: str, flaggers: str, start_page: str, stop_page: str) -> str: boardsDesc = "The boards on 4chan you wish to gather from." boards = unpack_http_get_list(boards) boards = parameter_blacklist(boards, 'boards', boardsDesc) @@ -237,7 +239,7 @@ def generate_csv(): start_page = request.args.get('start_page', None) stop_page = request.args.get('stop_page', None) - csvString = _generate_csv_string( + csvString = _generate_csv_string_4plebs( boards=boards, flaggers=flaggers, start_page=start_page, @@ -254,8 +256,8 @@ def generate_csv(): @app.route("/api/generate/4chan/json", methods=['GET']) -def generate_json(): - csvString = _generate_csv_string( +def generate_4plebs_json(): + csvString = _generate_csv_string_4plebs( boards=request.args.get('boards', None), flaggers=request.args.get('flaggers', None), start_page=request.args.get('start_page', None), From f3de4d1220174d9f8945de27c6ecc1cd810e5d54 Mon Sep 17 00:00:00 2001 From: Jay Date: Tue, 3 Dec 2019 00:39:43 -0600 Subject: [PATCH 02/42] updated routes for reddit configure page --- client/src/routes.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/client/src/routes.js b/client/src/routes.js index 029fdb4..75a5bcc 100644 --- a/client/src/routes.js +++ b/client/src/routes.js @@ -31,6 +31,7 @@ import DashboardPage from "views/Dashboard/Dashboard.js"; import MockDashboardPage from "views/DashboardMock/Dashboard.js"; import UserProfile from "views/UserProfile/UserProfile.js"; import Configure from "views/Configure/Configure.js"; +import ConfigureReddit from "views/ConfigureReddit/ConfigureReddit.js"; import TableList from "views/TableList/TableList.js"; import Typography from "views/Typography/Typography.js"; import Icons from "views/Icons/Icons.js"; @@ -58,6 +59,15 @@ const dashboardRoutes = [ component: Configure, layout: "/admin" }, + { + path: "/configure-reddit", + name: "(Beta) Configure Reddit", + rtlName: "ملف تعريفي للمستخدم", + icon: Build, + component: ConfigureReddit, + layout: "/admin" + }, + { // { // path: "/notifications", // name: "Notifications", From 34012ad86c812b7cca673a3f069fa4c2143aef10 Mon Sep 17 00:00:00 2001 From: Jay Date: Tue, 3 Dec 2019 00:40:02 -0600 Subject: [PATCH 03/42] updated 4chan routes --- client/src/routes.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/src/routes.js b/client/src/routes.js index 75a5bcc..59db564 100644 --- a/client/src/routes.js +++ b/client/src/routes.js @@ -45,7 +45,7 @@ import RTLPage from "views/RTLPage/RTLPage.js"; const dashboardRoutes = [ { path: "/dashboard", - name: "Dashboard", + name: "4Chan Dashboard", rtlName: "لوحة القيادة", icon: Dashboard, component: MockDashboardPage, @@ -53,7 +53,7 @@ const dashboardRoutes = [ }, { path: "/configure", - name: "Configure", + name: "Configure 4Chan", rtlName: "ملف تعريفي للمستخدم", icon: Build, component: Configure, From 1a86cb16b9f647d9db228ad3fc35616658a26258 Mon Sep 17 00:00:00 2001 From: Jay Date: Tue, 3 Dec 2019 00:41:16 -0600 Subject: [PATCH 04/42] created configure page --- .../views/ConfigureReddit/ConfigureReddit.js | 327 ++++++++++++++++++ client/src/views/ConfigureReddit/formModel.js | 75 ++++ 2 files changed, 402 insertions(+) create mode 100644 client/src/views/ConfigureReddit/ConfigureReddit.js create mode 100644 client/src/views/ConfigureReddit/formModel.js diff --git a/client/src/views/ConfigureReddit/ConfigureReddit.js b/client/src/views/ConfigureReddit/ConfigureReddit.js new file mode 100644 index 0000000..6669baa --- /dev/null +++ b/client/src/views/ConfigureReddit/ConfigureReddit.js @@ -0,0 +1,327 @@ +import React, { useState } from "react"; +// @material-ui/core components +import { makeStyles } from "@material-ui/core/styles"; +import InputLabel from "@material-ui/core/InputLabel"; +import FormControl from "@material-ui/core/FormControl"; +import TextField from "@material-ui/core/TextField"; +import Checkbox from "@material-ui/core/Checkbox"; +import Select from "@material-ui/core/Select"; +import FormControlLabel from "@material-ui/core/FormControlLabel"; +import Typography from "@material-ui/core/Typography"; +import Fade from "@material-ui/core/Fade"; + +// formik +import { Formik, Form, useField, Field } from "formik"; + +// core components +import GridItem from "components/Grid/GridItem.js"; +import GridContainer from "components/Grid/GridContainer.js"; +import Button from "components/CustomButtons/Button.js"; +import Card from "components/Card/Card.js"; +import CardHeader from "components/Card/CardHeader.js"; +import CardAvatar from "components/Card/CardAvatar.js"; +import CardBody from "components/Card/CardBody.js"; +import CardFooter from "components/Card/CardFooter.js"; +import Snackbar from "components/Notification/Notification.js"; + +// import avatar from "public/img/faces/bowser-img192.png"; + +import { inputsModels, initialValues, validationSchema } from "./formModel"; +import useHttp from "hooks/useHttp.hook"; +import inputStyles from "assets/jss/material-dashboard-react/components/customInputStyle.js"; +import classNames from "classnames"; +import { Paper, MenuItem } from "@material-ui/core"; + +import { useStoreActions } from "easy-peasy"; +import { useHistory } from "react-router-dom"; +import { conditionalExpression } from "@babel/types"; + +const styles = { + cardCategoryWhite: { + color: "rgba(255,255,255,.62)", + margin: "0", + fontSize: "14px", + marginTop: "0", + marginBottom: "0" + }, + cardTitleWhite: { + color: "#FFFFFF", + marginTop: "0px", + minHeight: "auto", + fontWeight: "300", + fontFamily: "'Roboto', 'Helvetica', 'Arial', sans-serif", + marginBottom: "3px", + textDecoration: "none" + }, + customInput: { + marginTop: "20px", + marginBottom: "15px" + }, + marginTopBot: { + marginTop: "35px", + marginBottom: "10px" + } +}; + +const MyCustomInput = ({ label, name, type, inputProps, component, id, value, menuItems }) => { + const useStyles = makeStyles(styles); + const classes = useStyles(); + + const [field, meta] = useField({ label, name, type }); + const errorText = meta.error && meta.touched ? meta.error : ""; + const error = !!errorText; + + const labelClasses = classNames({ + [" " + classes.labelRootError]: error, + [" " + classes.labelRootSuccess]: !error + }); + const underlineClasses = classNames({ + [classes.underlineError]: error, + [classes.underlineSuccess]: !error, + [classes.underline]: true + }); + const marginTop = classNames({ + [classes.marginTop]: label === undefined + }); + + inputProps = { + ...inputProps, + className: classes.customInput + }; + + switch (component) { + case "TextField": + return ( + + ); + case "Checkbox": + return ( + } + label={label} + /> + ); + case "Select": + return ( + + {menuItems.map(item => ( + {item.text} + ))} + + ); + } +}; + +const useStyles = makeStyles(styles); + +export default function UserProfile() { + const [showNotification, setShowNotification] = useState(false); + const [notification, setNotification] = useState({}); + const history = useHistory(); + const classes = useStyles(); + const fetchData = useStoreActions(actions => actions.reddit.fetchData); + + const close = () => { + setShowNotification(false); + }; + + const getRequestString = ({ + host, + port, + actionString, + boards, + flaggers, + startPage, + stopPage + }) => { + let requestString = ""; + + if (host) { + requestString = requestString + host; + } + if (port) { + requestString = requestString + `:${port}`; + } + if (actionString) { + requestString = requestString + actionString; + } + if (boards) { + requestString = requestString + `?boards=${boards.join(",")}`; + } + if (flaggers) { + requestString = requestString + `&flaggers=${flaggers.join(",")}`; + } + if (startPage) { + requestString = requestString + `&start_page=${startPage}`; + } + if (stopPage) { + requestString = requestString + `&stop_page=${stopPage}`; + } + + return requestString; + }; + + return ( + <> + {showNotification ? ( + + ) : ( + "" + )} + { + setSubmitting(true); + // make async call + // console.log(values); + try { + await fetchData({ + url: getRequestString(values), + method: "GET", + responseType: values.actionString.includes("csv") + ? "csv" + : values.actionString.includes("json") + ? "json" + : "text" + }); + await setNotification({ + type: "success", + message: "Request completed successfully" + }); + await setShowNotification(true); + setTimeout(function() { + setShowNotification(false); + }, 6000); + } catch (error) { + console.log(error, error.stack); + setNotification({ type: "danger", message: "Error with request, please try again" }); + setShowNotification(true); + setTimeout(function() { + setShowNotification(false); + }, 6000); + console.debug("Error posting try again"); + } + + setSubmitting(false); + }} + > + {({ values, errors, isSubmitting }) => ( + // Formik auto pass in onSubmit handler | onSubmit={handleSubmit} +
+ + + + +

Configure Reddit Query

+

Adjust settings

+ + + {/* Host Section */} + + + Host Configuration + + + + + {inputsModels.hostSection.map(field => { + const { columnSpan, formControlProps, ...rest } = field; + return ( + + + + + + ); + })} + + + {/* Request String */} + + + + Request String: {getRequestString(values)} + + + + + {notification.type === "success" ? ( + + + + + + ) : isSubmitting ? ( + + + + ) : Object.keys(errors).length > 0 ? ( + + + + ) : ( + + + + )} + + + {/* + + + e.preventDefault()}> + Bowser + + + +
CEO / CO-FOUNDER
+

Alec Thompson

+

+ Don{"'"}t be scared of the truth because we need to restart the human + foundation in truth And I love you like Kanye loves Kanye I love Rick Owens’ + bed design but the back is... +

+ + + + */} + + {/*
+ 
{JSON.stringify(values, null, 2)}
+ 
{JSON.stringify(errors, null, 2)}
+
*/} +
+ )} + + + ); +} diff --git a/client/src/views/ConfigureReddit/formModel.js b/client/src/views/ConfigureReddit/formModel.js new file mode 100644 index 0000000..3dea656 --- /dev/null +++ b/client/src/views/ConfigureReddit/formModel.js @@ -0,0 +1,75 @@ +import { TextField, Checkbox, Select } from "@material-ui/core"; +import * as yup from "yup"; + +const { REACT_APP_BOWSER_API_HOST, REACT_APP_BOWSER_API_PORT } = process.env; +const { REACT_APP_JARRON_API_HOST, REACT_APP_JARRON_API_PORT } = process.env; + +const initialValues = { + host: REACT_APP_BOWSER_API_HOST, + port: REACT_APP_BOWSER_API_PORT, + actionString: "/api/generate/reddit/json?subreddit=Sino" +}; + +const validationSchema = yup.object().shape({ + host: yup + .string("Host must be a number") + .max(40, "Host cannot exceed 21 characters") + .required("Host is required"), + port: yup + .number("Port must be a number") + .positive("Must be a postive number") + .required("Port is required"), + actionString: yup + .string("Action Page must be a string") + .max(40, "Action cannont be more than 40 characters") + .required("Action is required") +}); + +const inputsModels = { + hostSection: [ + { + id: "host-disabled", + name: "host", + label: "Host", + type: "input", + columnSpan: { xs: 12, sm: 12, md: 3 }, + formControlProps: { + fullWidth: true + }, + inputProps: { + disabled: true + }, + component: "TextField" + }, + { + id: "port-disabled", + name: "port", + label: "Port", + type: "input", + columnSpan: { xs: 12, sm: 12, md: 3 }, + formControlProps: { + fullWidth: true + }, + inputProps: { + disabled: true + }, + component: "TextField" + }, + { + id: "action-string", + name: "actionString", + label: "Action", + type: "input", + columnSpan: { xs: 12, sm: 12, md: 3 }, + formControlProps: { + fullWidth: true + }, + inputProps: { + disabled: true + }, + component: "TextField" + } + ] +}; + +export { inputsModels, initialValues, validationSchema }; From 9d414e7d963bd41d8971bf9f7643bb00ea3cd18c Mon Sep 17 00:00:00 2001 From: Jay Date: Tue, 3 Dec 2019 00:44:25 -0600 Subject: [PATCH 05/42] created reddit actions and state and modularized the store --- client/src/store/models/4chan.model.js | 404 +++++++++++++++++++++ client/src/store/models/index.js | 444 +----------------------- client/src/store/models/reddit.model.js | 71 ++++ 3 files changed, 479 insertions(+), 440 deletions(-) create mode 100644 client/src/store/models/4chan.model.js create mode 100644 client/src/store/models/reddit.model.js diff --git a/client/src/store/models/4chan.model.js b/client/src/store/models/4chan.model.js new file mode 100644 index 0000000..6c05110 --- /dev/null +++ b/client/src/store/models/4chan.model.js @@ -0,0 +1,404 @@ +import fetchHelper from "helpers/fetchHelper"; +import { thunk, action, actionOn, thunkOn, computed, debug } from "easy-peasy"; +import { tableHeaderNames } from "variables/general"; + +const postsModel = { + headers: [], + data: [], + boards: {}, + threads: {}, + postAnaylzed: 0, + terroismFlagCount: 0, + nsaPrismFlagCount: 0, + nsaEchelonFlagCount: 0, + hateSpeechFlagCount: 0, + conspiracyFlagCount: 0, + noContentFlagCount: 0, + racismFlagCount: 0, + dataType: "", + fetchData: thunk(async (actions, payload) => { + let data = await fetchHelper(payload); + const { responseType: dataType } = payload; + + if (dataType === "json") { + data = await { ...data }; + } + + // headers[headers.length - 1] = await headers[headers.length - 1].replace("\r", ""); + await actions.setDataType({ dataType }); + await actions.setPosts({ ...data }); // 👈 dispatch local actions to update state + actions.analyzeThreadIds({ ...data }); // 👈 dispatch local actions to update state + actions.analyzeBoards({ ...data }); // 👈 dispatch local actions to update state + actions.analyzeTerroismFlag({ ...data }); + actions.analyzeNsaPrismFlag({ ...data }); + actions.analyzeNsaEchelonFlag({ ...data }); + actions.analyzeHateSpeechFlag({ ...data }); + actions.analyzeRacismFlag({ ...data }); + actions.analyzeConspiracyFlag({ ...data }); + actions.analyzeNoContent({ ...data }); + if (dataType === "json") { + // actions.analyzeNsaPrismFlag({ ...data }); + } + }), + setDataType: action((state, payload) => { + const { dataType } = payload; + state.dataType = dataType; + }), + setPosts: action((state, payload) => { + var postAnaylzed = 0; + var formatteddata = []; + var headers = []; + + if (state.dataType === "csv") { + const { data, headers: incomingHeaders } = payload; + + formatteddata = data; + headers = incomingHeaders; + postAnaylzed = data.length; + } + + // json request + if (state.dataType === "json") { + console.log("PAYLOAD", payload); + let data = []; + postAnaylzed = Object.keys(payload.board).length; + // const { data, headers } = payload; + + for (let index = 0; index < postAnaylzed; index++) { + let obj = {}; + + obj[tableHeaderNames.NoContent] = payload[tableHeaderNames.NoContent][index]; + if (payload[tableHeaderNames.ContentFlaggerNsaEchelon]) { + obj[tableHeaderNames.ContentFlaggerNsaEchelon] = + payload[tableHeaderNames.ContentFlaggerNsaEchelon][index]; + } + if (payload[tableHeaderNames.ContentFlaggerHateSpeech]) { + obj[tableHeaderNames.ContentFlaggerHateSpeech] = + payload[tableHeaderNames.ContentFlaggerHateSpeech][index]; + } + if (payload[tableHeaderNames.ContentFlaggerNsaPrism]) { + obj[tableHeaderNames.ContentFlaggerNsaPrism] = + payload[tableHeaderNames.ContentFlaggerNsaPrism][index]; + } + if (payload[tableHeaderNames.ContentFlaggerRacism]) { + obj[tableHeaderNames.ContentFlaggerRacism] = + payload[tableHeaderNames.ContentFlaggerRacism][index]; + } + if (payload[tableHeaderNames.ContentFlaggerConspiracy]) { + obj[tableHeaderNames.ContentFlaggerConspiracy] = + payload[tableHeaderNames.ContentFlaggerConspiracy][index]; + } + if (payload[tableHeaderNames.ContentFlaggerTerroism]) { + obj[tableHeaderNames.ContentFlaggerTerroism] = + payload[tableHeaderNames.ContentFlaggerTerroism][index]; + } + + obj.board = payload.board[index]; + obj.country_code = payload.country_code[index]; + obj.full_comment = payload.full_comment[index]; + obj.op = payload.op[index]; + obj.post_api_url = payload.post_api_url[index]; + obj.post_id = payload.post_id[index]; + obj.post_url = payload.post_url[index]; + obj.thread_api_url = payload.thread_api_url[index]; + obj.thread_id = payload.thread_id[index]; + obj.thread_url = payload.thread_url[index]; + obj.timestamp_ISO8601 = payload.timestamp_ISO8601[index]; + obj.timestamp_epoch = payload.timestamp_epoch[index]; + data.push(obj); + } + + formatteddata = data; + headers = Object.keys(payload); + postAnaylzed = postAnaylzed; + } + + state.postAnaylzed = postAnaylzed; + state.data = formatteddata; + state.headers = headers; + }), + + analyzeThreadIds: action((state, payload) => { + let threads = {}; + + state.data.forEach(post => { + const { thread_id } = post; + threads[thread_id] = threads[thread_id] + 1 || 1; + }); + + state.threads = threads; + state.threads.count = Object.keys(threads).length; + }), + analyzeBoards: action((state, payload) => { + let boards = {}; + + state.data.forEach(post => { + const { board } = post; + boards[board] = boards[board] + 1 || 1; + }); + + state.boards = boards; + state.boards.count = Object.keys(boards).length; + }), + analyzeTerroismFlag: action((state, payload) => { + let count = 0; + let feild = ""; + + if ( + typeof state.data[0][tableHeaderNames.ContentFlaggerTerroism] === "boolean" || + typeof state.data[0][tableHeaderNames.ContentFlaggerTerroism] === "string" + ) { + feild = tableHeaderNames.ContentFlaggerTerroism; + } else if ( + typeof state.data[0][tableHeaderNames.ContentFlaggerTerroism + "\r"] === "boolean" || + typeof state.data[0][tableHeaderNames.ContentFlaggerTerroism + "\r"] === "string" + ) { + feild = tableHeaderNames.ContentFlaggerTerroism + "\r"; + } + + if (feild) { + state.data.forEach((post, index) => { + if (state.dataType === "csv") { + if (post[feild].includes("True")) { + count = count + 1 || 1; + } + } + + if (state.dataType === "json") { + if (post[feild] == true) { + count = count + 1 || 1; + state.data[index][feild] = "True"; + } else { + state.data[index][feild] = "False"; + } + } + }); + } + + state.terroismFlagCount = count; + }), + analyzeNsaPrismFlag: action((state, payload) => { + let count = 0; + let feild = ""; + + if ( + typeof state.data[0][tableHeaderNames.ContentFlaggerNsaPrism] === "boolean" || + typeof state.data[0][tableHeaderNames.ContentFlaggerNsaPrism] === "string" + ) { + feild = tableHeaderNames.ContentFlaggerNsaPrism; + } else if ( + typeof state.data[0][tableHeaderNames.ContentFlaggerNsaPrism + "\r"] === "boolean" || + typeof state.data[0][tableHeaderNames.ContentFlaggerNsaPrism + "\r"] === "string" + ) { + feild = tableHeaderNames.ContentFlaggerNsaPrism + "\r"; + } + + if (feild) { + state.data.forEach((post, index) => { + if (state.dataType === "csv") { + if (post[feild].includes("True")) { + count = count + 1 || 1; + } + } + + if (state.dataType === "json") { + if (post[feild] == true) { + count = count + 1 || 1; + state.data[index][feild] = "True"; + } else { + state.data[index][feild] = "False"; + } + } + }); + + state.nsaPrismFlagCount = count; + } + }), + analyzeNsaEchelonFlag: action((state, payload) => { + let count = 0; + let feild = ""; + + if ( + typeof state.data[0][tableHeaderNames.ContentFlaggerNsaEchelon] === "boolean" || + typeof state.data[0][tableHeaderNames.ContentFlaggerNsaEchelon] === "string" + ) { + feild = tableHeaderNames.ContentFlaggerNsaEchelon; + } else if ( + typeof state.data[0][tableHeaderNames.ContentFlaggerNsaEchelon + "\r"] === "boolean" || + typeof state.data[0][tableHeaderNames.ContentFlaggerNsaEchelon + "\r"] === "string" + ) { + feild = tableHeaderNames.ContentFlaggerNsaEchelon + "\r"; + } + + if (feild) { + state.data.forEach((post, index) => { + if (state.dataType === "csv") { + if (post[feild].includes("True")) { + count = count + 1 || 1; + } + } + + if (state.dataType === "json") { + if (post[feild] == true) { + count = count + 1 || 1; + state.data[index][feild] = "True"; + } else { + state.data[index][feild] = "False"; + } + } + }); + + state.nsaEchelonFlagCount = count; + } + }), + analyzeHateSpeechFlag: action((state, payload) => { + let count = 0; + let feild = ""; + + if ( + typeof state.data[0][tableHeaderNames.ContentFlaggerHateSpeech] === "boolean" || + typeof state.data[0][tableHeaderNames.ContentFlaggerHateSpeech] === "string" + ) { + feild = tableHeaderNames.ContentFlaggerHateSpeech; + } else if ( + typeof state.data[0][tableHeaderNames.ContentFlaggerHateSpeech + "\r"] === "boolean" || + typeof state.data[0][tableHeaderNames.ContentFlaggerHateSpeech + "\r"] === "string" + ) { + feild = tableHeaderNames.ContentFlaggerHateSpeech + "\r"; + } + + if (feild) { + state.data.forEach((post, index) => { + if (state.dataType === "csv") { + if (post[feild].includes("True")) { + count = count + 1 || 1; + } + } + + if (state.dataType === "json") { + if (post[feild] == true) { + count = count + 1 || 1; + state.data[index][feild] = "True"; + } else { + state.data[index][feild] = "False"; + } + } + }); + + state.hateSpeechFlagCount = count; + } + }), + analyzeRacismFlag: action((state, payload) => { + let count = 0; + let feild = ""; + + if ( + typeof state.data[0][tableHeaderNames.ContentFlaggerRacism] === "boolean" || + typeof state.data[0][tableHeaderNames.ContentFlaggerRacism] === "string" + ) { + feild = tableHeaderNames.ContentFlaggerRacism; + } else if ( + typeof state.data[0][tableHeaderNames.ContentFlaggerRacism + "\r"] === "boolean" || + typeof state.data[0][tableHeaderNames.ContentFlaggerRacism + "\r"] === "string" + ) { + feild = tableHeaderNames.ContentFlaggerRacism + "\r"; + } + + if (feild) { + state.data.forEach((post, index) => { + if (state.dataType === "csv") { + if (post[feild].includes("True")) { + count = count + 1 || 1; + } + } + + if (state.dataType === "json") { + if (post[feild] == true) { + count = count + 1 || 1; + state.data[index][feild] = "True"; + } else { + state.data[index][feild] = "False"; + } + } + }); + + state.racismFlagCount = count; + } + }), + analyzeConspiracyFlag: action((state, payload) => { + let count = 0; + let feild = ""; + + if ( + typeof state.data[0][tableHeaderNames.ContentFlaggerConspiracy] === "boolean" || + typeof state.data[0][tableHeaderNames.ContentFlaggerConspiracy] === "string" + ) { + feild = tableHeaderNames.ContentFlaggerConspiracy; + } else if ( + typeof state.data[0][tableHeaderNames.ContentFlaggerConspiracy + "\r"] === "boolean" || + typeof state.data[0][tableHeaderNames.ContentFlaggerConspiracy + "\r"] === "string" + ) { + feild = tableHeaderNames.ContentFlaggerConspiracy + "\r"; + } + + if (feild) { + state.data.forEach((post, index) => { + if (state.dataType === "csv") { + if (post[feild].includes("True")) { + count = count + 1 || 1; + } + } + + if (state.dataType === "json") { + if (post[feild] == true) { + count = count + 1 || 1; + state.data[index][feild] = "True"; + } else { + state.data[index][feild] = "False"; + } + } + }); + + state.conspiracyFlagCount = count; + } + }), + analyzeNoContent: action((state, payload) => { + let count = 0; + let feild = ""; + + if ( + typeof state.data[0][tableHeaderNames.NoContent] === "boolean" || + typeof state.data[0][tableHeaderNames.NoContent] === "string" + ) { + feild = tableHeaderNames.NoContent; + } else if ( + typeof state.data[0][tableHeaderNames.NoContent + "\r"] === "boolean" || + typeof state.data[0][tableHeaderNames.NoContent + "\r"] === "string" + ) { + feild = tableHeaderNames.NoContent + "\r"; + } + + if (feild) { + state.data.forEach((post, index) => { + if (state.dataType === "csv") { + if (post[feild].includes("True")) { + count = count + 1 || 1; + } + } + + if (state.dataType === "json") { + if (post[feild] == true) { + count = count + 1 || 1; + state.data[index][feild] = "True"; + } else { + state.data[index][feild] = "False"; + } + } + }); + + state.noContentFlagCount = count; + } + }) +}; + +export default postsModel; diff --git a/client/src/store/models/index.js b/client/src/store/models/index.js index f9502e9..176b566 100644 --- a/client/src/store/models/index.js +++ b/client/src/store/models/index.js @@ -1,443 +1,7 @@ -import fetchHelper from "helpers/fetchHelper"; -import { thunk, action, actionOn, thunkOn, computed, debug } from "easy-peasy"; -import { tableHeaderNames } from "variables/general"; - -const postsModel = { - headers: [], - data: [], - boards: {}, - threads: {}, - postAnaylzed: 0, - terroismFlagCount: 0, - nsaPrismFlagCount: 0, - nsaEchelonFlagCount: 0, - hateSpeechFlagCount: 0, - conspiracyFlagCount: 0, - noContentFlagCount: 0, - racismFlagCount: 0, - dataType: "", - fetchData: thunk(async (actions, payload) => { - console.log(payload); - let data = await fetchHelper(payload); - const { responseType: dataType } = payload; - - if (dataType === "json") { - data = await { ...data }; - } - - // headers[headers.length - 1] = await headers[headers.length - 1].replace("\r", ""); - await actions.setDataType({ dataType }); - await actions.setPosts({ ...data }); // 👈 dispatch local actions to update state - actions.analyzeThreadIds({ ...data }); // 👈 dispatch local actions to update state - actions.analyzeBoards({ ...data }); // 👈 dispatch local actions to update state - actions.analyzeTerroismFlag({ ...data }); - actions.analyzeNsaPrismFlag({ ...data }); - actions.analyzeNsaEchelonFlag({ ...data }); - actions.analyzeHateSpeechFlag({ ...data }); - actions.analyzeRacismFlag({ ...data }); - actions.analyzeConspiracyFlag({ ...data }); - actions.analyzeNoContent({ ...data }); - if (dataType === "json") { - // actions.analyzeNsaPrismFlag({ ...data }); - } - }), - setDataType: action((state, payload) => { - const { dataType } = payload; - state.dataType = dataType; - }), - setPosts: action((state, payload) => { - var postAnaylzed = 0; - var formatteddata = []; - var headers = []; - - if (state.dataType === "csv") { - const { data, headers: incomingHeaders } = payload; - - formatteddata = data; - headers = incomingHeaders; - postAnaylzed = data.length; - } - - // json request - if (state.dataType === "json") { - console.log("PAYLOAD", payload); - let data = []; - postAnaylzed = Object.keys(payload.board).length; - // const { data, headers } = payload; - - for (let index = 0; index < postAnaylzed; index++) { - let obj = {}; - - obj[tableHeaderNames.NoContent] = payload[tableHeaderNames.NoContent][index]; - if (payload[tableHeaderNames.ContentFlaggerNsaEchelon]) { - obj[tableHeaderNames.ContentFlaggerNsaEchelon] = - payload[tableHeaderNames.ContentFlaggerNsaEchelon][index]; - } - if (payload[tableHeaderNames.ContentFlaggerHateSpeech]) { - obj[tableHeaderNames.ContentFlaggerHateSpeech] = - payload[tableHeaderNames.ContentFlaggerHateSpeech][index]; - } - if (payload[tableHeaderNames.ContentFlaggerNsaPrism]) { - obj[tableHeaderNames.ContentFlaggerNsaPrism] = - payload[tableHeaderNames.ContentFlaggerNsaPrism][index]; - } - if (payload[tableHeaderNames.ContentFlaggerRacism]) { - obj[tableHeaderNames.ContentFlaggerRacism] = - payload[tableHeaderNames.ContentFlaggerRacism][index]; - } - if (payload[tableHeaderNames.ContentFlaggerConspiracy]) { - obj[tableHeaderNames.ContentFlaggerConspiracy] = - payload[tableHeaderNames.ContentFlaggerConspiracy][index]; - } - if (payload[tableHeaderNames.ContentFlaggerTerroism]) { - obj[tableHeaderNames.ContentFlaggerTerroism] = - payload[tableHeaderNames.ContentFlaggerTerroism][index]; - } - - obj.board = payload.board[index]; - obj.country_code = payload.country_code[index]; - obj.full_comment = payload.full_comment[index]; - obj.op = payload.op[index]; - obj.post_api_url = payload.post_api_url[index]; - obj.post_id = payload.post_id[index]; - obj.post_url = payload.post_url[index]; - obj.thread_api_url = payload.thread_api_url[index]; - obj.thread_id = payload.thread_id[index]; - obj.thread_url = payload.thread_url[index]; - obj.timestamp_ISO8601 = payload.timestamp_ISO8601[index]; - obj.timestamp_epoch = payload.timestamp_epoch[index]; - data.push(obj); - } - - formatteddata = data; - headers = Object.keys(payload); - postAnaylzed = postAnaylzed; - } - - state.postAnaylzed = postAnaylzed; - state.data = formatteddata; - state.headers = headers; - }), - - analyzeThreadIds: action((state, payload) => { - let threads = {}; - - state.data.forEach(post => { - const { thread_id } = post; - threads[thread_id] = threads[thread_id] + 1 || 1; - }); - - state.threads = threads; - state.threads.count = Object.keys(threads).length; - }), - analyzeBoards: action((state, payload) => { - let boards = {}; - - state.data.forEach(post => { - const { board } = post; - boards[board] = boards[board] + 1 || 1; - }); - - state.boards = boards; - state.boards.count = Object.keys(boards).length; - }), - analyzeTerroismFlag: action((state, payload) => { - let count = 0; - let feild = ""; - - if ( - typeof state.data[0][tableHeaderNames.ContentFlaggerTerroism] === "boolean" || - typeof state.data[0][tableHeaderNames.ContentFlaggerTerroism] === "string" - ) { - feild = tableHeaderNames.ContentFlaggerTerroism; - } else if ( - typeof state.data[0][tableHeaderNames.ContentFlaggerTerroism + "\r"] === "boolean" || - typeof state.data[0][tableHeaderNames.ContentFlaggerTerroism + "\r"] === "string" - ) { - feild = tableHeaderNames.ContentFlaggerTerroism + "\r"; - } - - if (feild) { - state.data.forEach((post, index) => { - if (state.dataType === "csv") { - if (post[feild].includes("True")) { - count = count + 1 || 1; - } - } - - if (state.dataType === "json") { - if (post[feild] == true) { - count = count + 1 || 1; - state.data[index][feild] = "True"; - } else { - state.data[index][feild] = "False"; - } - } - }); - } - - state.terroismFlagCount = count; - }), - analyzeNsaPrismFlag: action((state, payload) => { - let count = 0; - let feild = ""; - - if ( - typeof state.data[0][tableHeaderNames.ContentFlaggerNsaPrism] === "boolean" || - typeof state.data[0][tableHeaderNames.ContentFlaggerNsaPrism] === "string" - ) { - feild = tableHeaderNames.ContentFlaggerNsaPrism; - } else if ( - typeof state.data[0][tableHeaderNames.ContentFlaggerNsaPrism + "\r"] === "boolean" || - typeof state.data[0][tableHeaderNames.ContentFlaggerNsaPrism + "\r"] === "string" - ) { - feild = tableHeaderNames.ContentFlaggerNsaPrism + "\r"; - } - - if (feild) { - state.data.forEach((post, index) => { - if (state.dataType === "csv") { - if (post[feild].includes("True")) { - count = count + 1 || 1; - } - } - - if (state.dataType === "json") { - if (post[feild] == true) { - count = count + 1 || 1; - state.data[index][feild] = "True"; - } else { - state.data[index][feild] = "False"; - } - } - }); - - state.nsaPrismFlagCount = count; - } - }), - analyzeNsaEchelonFlag: action((state, payload) => { - let count = 0; - let feild = ""; - - if ( - typeof state.data[0][tableHeaderNames.ContentFlaggerNsaEchelon] === "boolean" || - typeof state.data[0][tableHeaderNames.ContentFlaggerNsaEchelon] === "string" - ) { - feild = tableHeaderNames.ContentFlaggerNsaEchelon; - } else if ( - typeof state.data[0][tableHeaderNames.ContentFlaggerNsaEchelon + "\r"] === "boolean" || - typeof state.data[0][tableHeaderNames.ContentFlaggerNsaEchelon + "\r"] === "string" - ) { - feild = tableHeaderNames.ContentFlaggerNsaEchelon + "\r"; - } - - if (feild) { - state.data.forEach((post, index) => { - if (state.dataType === "csv") { - if (post[feild].includes("True")) { - count = count + 1 || 1; - } - } - - if (state.dataType === "json") { - if (post[feild] == true) { - count = count + 1 || 1; - state.data[index][feild] = "True"; - } else { - state.data[index][feild] = "False"; - } - } - }); - - state.nsaEchelonFlagCount = count; - } - }), - analyzeHateSpeechFlag: action((state, payload) => { - let count = 0; - let feild = ""; - - if ( - typeof state.data[0][tableHeaderNames.ContentFlaggerHateSpeech] === "boolean" || - typeof state.data[0][tableHeaderNames.ContentFlaggerHateSpeech] === "string" - ) { - feild = tableHeaderNames.ContentFlaggerHateSpeech; - } else if ( - typeof state.data[0][tableHeaderNames.ContentFlaggerHateSpeech + "\r"] === "boolean" || - typeof state.data[0][tableHeaderNames.ContentFlaggerHateSpeech + "\r"] === "string" - ) { - feild = tableHeaderNames.ContentFlaggerHateSpeech + "\r"; - } - - if (feild) { - state.data.forEach((post, index) => { - if (state.dataType === "csv") { - if (post[feild].includes("True")) { - count = count + 1 || 1; - } - } - - if (state.dataType === "json") { - if (post[feild] == true) { - count = count + 1 || 1; - state.data[index][feild] = "True"; - } else { - state.data[index][feild] = "False"; - } - } - }); - - state.hateSpeechFlagCount = count; - } - }), - analyzeRacismFlag: action((state, payload) => { - let count = 0; - let feild = ""; - - if ( - typeof state.data[0][tableHeaderNames.ContentFlaggerRacism] === "boolean" || - typeof state.data[0][tableHeaderNames.ContentFlaggerRacism] === "string" - ) { - feild = tableHeaderNames.ContentFlaggerRacism; - } else if ( - typeof state.data[0][tableHeaderNames.ContentFlaggerRacism + "\r"] === "boolean" || - typeof state.data[0][tableHeaderNames.ContentFlaggerRacism + "\r"] === "string" - ) { - feild = tableHeaderNames.ContentFlaggerRacism + "\r"; - } - - if (feild) { - state.data.forEach((post, index) => { - if (state.dataType === "csv") { - if (post[feild].includes("True")) { - count = count + 1 || 1; - } - } - - if (state.dataType === "json") { - if (post[feild] == true) { - count = count + 1 || 1; - state.data[index][feild] = "True"; - } else { - state.data[index][feild] = "False"; - } - } - }); - - state.racismFlagCount = count; - } - }), - analyzeConspiracyFlag: action((state, payload) => { - let count = 0; - let feild = ""; - - if ( - typeof state.data[0][tableHeaderNames.ContentFlaggerConspiracy] === "boolean" || - typeof state.data[0][tableHeaderNames.ContentFlaggerConspiracy] === "string" - ) { - feild = tableHeaderNames.ContentFlaggerConspiracy; - } else if ( - typeof state.data[0][tableHeaderNames.ContentFlaggerConspiracy + "\r"] === "boolean" || - typeof state.data[0][tableHeaderNames.ContentFlaggerConspiracy + "\r"] === "string" - ) { - feild = tableHeaderNames.ContentFlaggerConspiracy + "\r"; - } - - if (feild) { - state.data.forEach((post, index) => { - if (state.dataType === "csv") { - if (post[feild].includes("True")) { - count = count + 1 || 1; - } - } - - if (state.dataType === "json") { - if (post[feild] == true) { - count = count + 1 || 1; - state.data[index][feild] = "True"; - } else { - state.data[index][feild] = "False"; - } - } - }); - - state.conspiracyFlagCount = count; - } - }), - analyzeNoContent: action((state, payload) => { - let count = 0; - let feild = ""; - - if ( - typeof state.data[0][tableHeaderNames.NoContent] === "boolean" || - typeof state.data[0][tableHeaderNames.NoContent] === "string" - ) { - feild = tableHeaderNames.NoContent; - } else if ( - typeof state.data[0][tableHeaderNames.NoContent + "\r"] === "boolean" || - typeof state.data[0][tableHeaderNames.NoContent + "\r"] === "string" - ) { - feild = tableHeaderNames.NoContent + "\r"; - } - - if (feild) { - state.data.forEach((post, index) => { - if (state.dataType === "csv") { - if (post[feild].includes("True")) { - count = count + 1 || 1; - } - } - - if (state.dataType === "json") { - if (post[feild] == true) { - count = count + 1 || 1; - state.data[index][feild] = "True"; - } else { - state.data[index][feild] = "False"; - } - } - }); - - state.noContentFlagCount = count; - } - }) -}; - -const basketModel = { - productIds: [1], - logs: [], - items: { - 1: { id: 1, name: "Peas", price: 10 } - }, - - // async - updateProduct: thunk(async (actions, payload) => { - // const updated = await productService.update(payload.id, payload); - const updated = { id: 1, name: "jay" }; - - actions.setProduct(updated); // 👈 dispatch local actions to update state - }), - // action - setProduct: action((state, payload) => { - state.items[payload.id] = payload; - }), - addProduct: action((state, payload) => { - state.productIds.push(payload); - }), - // computed values - count: computed(state => Object.values(state.items).length), - - // lisenterns - onAddedToBasket: actionOn( - // targetResolver function receives actions and resolves the targets: - (actions, storeActions) => storeActions.basket.addProduct, - // action handler that executes when target is executed: - (state, target) => { - state.logs.push(`Added product ${target.payload} to basket`); - } - ) -}; +import postsModel from "./4chan.model"; +import redditModel from "./reddit.model"; export const storeModel = { - basket: basketModel, - posts: postsModel + posts: postsModel, + reddit: redditModel }; diff --git a/client/src/store/models/reddit.model.js b/client/src/store/models/reddit.model.js new file mode 100644 index 0000000..c14e47b --- /dev/null +++ b/client/src/store/models/reddit.model.js @@ -0,0 +1,71 @@ +import fetchHelper from "helpers/fetchHelper"; +import { thunk, action, actionOn, thunkOn, computed, debug } from "easy-peasy"; +import { tableHeaderNames } from "variables/general"; + +const postsModel = { + headers: [], + data: [], + threads: [], + postAnaylzed: 0, + dataType: "", + fetchData: thunk(async (actions, payload) => { + let data = await fetchHelper(payload); + const { responseType: dataType } = payload; + + if (dataType === "json") { + data = await { ...data }; + } + + await actions.setDataType({ dataType }); + await actions.setPosts({ ...data }); // 👈 dispatch local actions to update state + }), + setDataType: action((state, payload) => { + const { dataType } = payload; + state.dataType = dataType; + }), + setPosts: action((state, payload) => { + var postAnaylzed = 0; + var formatteddata = []; + + // json request + if (state.dataType === "json") { + payload.subreddit_response.data.children.forEach(post => { + const { data } = post; + let { author, media_only, created, over_18, selftext, title, url, ups, downs } = data; + + if (over_18) { + over_18 = "True"; + } else { + over_18 = "False"; + } + + if (media_only) { + media_only = "True"; + } else { + media_only = "False"; + } + + let obj = {}; + + obj.author = author; + obj.media_only = media_only; + obj.created = created; + obj.over_18 = over_18; + obj.selftext = selftext; + obj.title = title; + obj.url = url; + obj.ups = ups; + obj.downs = downs; + + formatteddata.push(obj); + postAnaylzed++; + }); + } + + state.headers = Object.keys(formatteddata[0]); + state.data = formatteddata; + state.threads = payload.thread_list; + }) +}; + +export default postsModel; From c62f3cea2da1d79b70eb991adafd23462854ef86 Mon Sep 17 00:00:00 2001 From: Jay Date: Tue, 3 Dec 2019 00:45:11 -0600 Subject: [PATCH 06/42] updated fetch helper --- client/src/helpers/fetchHelper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/helpers/fetchHelper.js b/client/src/helpers/fetchHelper.js index 1d2fbd3..0e68aac 100644 --- a/client/src/helpers/fetchHelper.js +++ b/client/src/helpers/fetchHelper.js @@ -21,7 +21,7 @@ const fetchHelper = async ({ } if (!response.ok) { - throw new Error("Could not fetch person!"); + throw new Error("Could not fetch item!"); } } catch (error) { console.error(error); From a3c1362044e20f77413037f63a692d5b258265c0 Mon Sep 17 00:00:00 2001 From: Jay Date: Tue, 3 Dec 2019 00:45:33 -0600 Subject: [PATCH 07/42] added reddit dashboard route --- client/src/routes.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/client/src/routes.js b/client/src/routes.js index 59db564..7547efe 100644 --- a/client/src/routes.js +++ b/client/src/routes.js @@ -29,6 +29,7 @@ import AccessibilityNew from "@material-ui/icons/AccessibilityNew"; // core components/views for Admin layout import DashboardPage from "views/Dashboard/Dashboard.js"; import MockDashboardPage from "views/DashboardMock/Dashboard.js"; +import DashboardRedditPage from "views/DashboardReddit/Dashboard.js"; import UserProfile from "views/UserProfile/UserProfile.js"; import Configure from "views/Configure/Configure.js"; import ConfigureReddit from "views/ConfigureReddit/ConfigureReddit.js"; @@ -68,6 +69,13 @@ const dashboardRoutes = [ layout: "/admin" }, { + path: "/dashboard-reddit", + name: "(Beta) Reddit Dashboard", + rtlName: "ملف تعريفي للمستخدم", + icon: Build, + component: DashboardRedditPage, + layout: "/admin" + }, // { // path: "/notifications", // name: "Notifications", From ac10edf5f4301b187206084387a33e8adbdba6bf Mon Sep 17 00:00:00 2001 From: Jay Date: Tue, 3 Dec 2019 00:46:12 -0600 Subject: [PATCH 08/42] created a more precise expression for active route classes --- client/src/components/Sidebar/Sidebar.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/src/components/Sidebar/Sidebar.js b/client/src/components/Sidebar/Sidebar.js index 1563992..17f0405 100644 --- a/client/src/components/Sidebar/Sidebar.js +++ b/client/src/components/Sidebar/Sidebar.js @@ -18,12 +18,15 @@ import RTLNavbarLinks from "components/Navbars/RTLNavbarLinks.js"; import styles from "assets/jss/material-dashboard-react/components/sidebarStyle.js"; const useStyles = makeStyles(styles); +import { useHistory } from "react-router-dom"; export default function Sidebar(props) { const classes = useStyles(); + const history = useHistory(); + // verifies if routeName is the one active (in browser input) function activeRoute(routeName) { - return window.location.href.indexOf(routeName) > -1 ? true : false; + return history.location.pathname === routeName ? true : false; } const { color, logo, image, logoText, routes } = props; var links = ( From cb32fb825dfccd9c01aa7b02a2b5c43fcdddaa84 Mon Sep 17 00:00:00 2001 From: Jay Date: Tue, 3 Dec 2019 00:48:22 -0600 Subject: [PATCH 09/42] updated 4chan form to use new /api/4chan routes --- client/src/views/Configure/formModel.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/src/views/Configure/formModel.js b/client/src/views/Configure/formModel.js index b74c3ac..e206f1a 100644 --- a/client/src/views/Configure/formModel.js +++ b/client/src/views/Configure/formModel.js @@ -6,7 +6,7 @@ const { REACT_APP_BOWSER_API_HOST, REACT_APP_BOWSER_API_PORT } = process.env; const initialValues = { host: REACT_APP_BOWSER_API_HOST, port: REACT_APP_BOWSER_API_PORT, - actionString: "/generate/csv", + actionString: "/api/generate/4chan/csv", boards: [], flaggers: [], startPage: 1, @@ -16,7 +16,7 @@ const initialValues = { const validationSchema = yup.object().shape({ host: yup .string("Host must be a number") - .max(21, "Host cannot exceed 21 characters") + .max(40, "Host cannot exceed 21 characters") .required("Host is required"), port: yup .number("Port must be a number") @@ -24,7 +24,7 @@ const validationSchema = yup.object().shape({ .required("Port is required"), actionString: yup .string("Action Page must be a string") - .max(14, "Action cannont be more than 14 characters") + .max(24, "Action cannont be more than 14 characters") .required("Action is required"), boards: yup .array() @@ -82,8 +82,8 @@ const inputsModels = { id: "action-string", name: "actionString", menuItems: [ - { value: "/generate/csv", text: "Generate CSV" }, - { value: "/generate/json", text: "Generate JSON" } + { value: "/api/generate/4chan/csv", text: "Generate CSV" }, + { value: "/api/generate/4chan/json", text: "Generate JSON" } ], columnSpan: { xs: 12, sm: 12, md: 4 }, formControlProps: { From e10f7f00af7f9022386082a017dbe33b01702365 Mon Sep 17 00:00:00 2001 From: Jay Date: Tue, 3 Dec 2019 00:48:58 -0600 Subject: [PATCH 10/42] updated 4chan form to be full width for md and up screens --- client/src/views/Configure/Configure.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/client/src/views/Configure/Configure.js b/client/src/views/Configure/Configure.js index 2f61a67..3755287 100644 --- a/client/src/views/Configure/Configure.js +++ b/client/src/views/Configure/Configure.js @@ -221,7 +221,7 @@ export default function UserProfile() { // Formik auto pass in onSubmit handler | onSubmit={handleSubmit}
- +

Configure Data Query

@@ -365,6 +365,12 @@ export default function UserProfile() { + ) : Object.keys(errors).length > 0 ? ( + + + ) : ( - ) : isSubmitting ? ( - - - - ) : Object.keys(errors).length > 0 ? ( - - - + ) : isSubmitting && !isValidating ? ( + + + + + + ) : !isValid ? ( + + + + + + ) : !dirty ? ( + + + + + ) : ( @@ -278,12 +261,14 @@ export default function UserProfile() { - ) : Object.keys(errors).length > 0 ? ( - - - + ) : !isValid ? ( + + + + + ) : (