describe of  EMA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "os.chdir(r'G:\\main\\data\\Data Mining\\College Experience Study Dataset\\EMA')\n",
    "data = pd.read_csv('general_ema.csv')\n",
    "data_main = data[np.isnan(data['pam'])]\n",
    "ind = list(data_main.index)\n",
    "data_main = data.drop(index=ind,axis=0)\n",
    "data_main = data_main.reset_index()\n",
    "data_main = data_main.drop(['index'], axis=1)\n",
    "EMA = data_main[['uid','day','pam', 'phq4_score', 'social_level', 'stress']]\n",
    "describe = EMA.describe()\n",
    "describe = describe.drop(['day'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(dpi=80,figsize=(16,10))\n",
    "plt.subplot(2,4,1)\n",
    "x = ['pam', 'phq4', 'social', 'stress']\n",
    "y = list(describe.loc['count', :])\n",
    "plt.scatter(x,y)\n",
    "plt.title('count', loc='left')\n",
    "\n",
    "plt.subplot(2,4,2)\n",
    "y = list(describe.loc['mean', :])\n",
    "plt.scatter(x,y)\n",
    "plt.title('mean', loc='left')\n",
    "\n",
    "plt.subplot(2,4,3)\n",
    "y = list(describe.loc['std', :])\n",
    "plt.scatter(x,y)\n",
    "plt.title('std', loc='left')\n",
    "\n",
    "plt.subplot(2,4,4)\n",
    "y = list(describe.loc['min', :])\n",
    "plt.scatter(x,y)\n",
    "plt.title('min', loc='left')\n",
    "\n",
    "plt.subplot(2,4,5)\n",
    "y = list(describe.loc['25%', :])\n",
    "plt.scatter(x,y)\n",
    "plt.title('25%', loc='left')\n",
    "\n",
    "plt.subplot(2,4,6)\n",
    "y = list(describe.loc['50%', :])\n",
    "plt.scatter(x,y)\n",
    "plt.title('50%', loc='left')\n",
    "\n",
    "plt.subplot(2,4,7)\n",
    "y = list(describe.loc['75%', :])\n",
    "plt.scatter(x,y)\n",
    "plt.title('75%', loc='left')\n",
    "\n",
    "plt.subplot(2,4,8)\n",
    "y = list(describe.loc['max', :])\n",
    "plt.scatter(x,y)\n",
    "plt.title('max', loc='left')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(dpi=80,figsize=(10,10))\n",
    "plt.subplot(2,2,1)\n",
    "plt.hist(list(EMA['pam']),10)\n",
    "plt.title('pam')\n",
    "plt.subplot(2,2,2)\n",
    "plt.hist(list(EMA['phq4_score']),10)\n",
    "plt.title('phq4')\n",
    "plt.subplot(2,2,3)\n",
    "plt.hist(list(EMA['social_level']),5)\n",
    "plt.title('social')\n",
    "plt.subplot(2,2,4)\n",
    "plt.hist(list(EMA['stress']),5)\n",
    "plt.title('stress')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "testee_pam_median = []\n",
    "testee_phq4_median = []\n",
    "testee_pam_std = []\n",
    "testee_phq4_std = []\n",
    "for x in set(EMA['uid']):\n",
    "    testee = EMA[EMA['uid'] == x]\n",
    "    testee = testee.describe()\n",
    "    testee_pam_median.append(testee.loc['50%', 'pam'])\n",
    "    testee_phq4_median.append(testee.loc['50%', 'phq4_score'])\n",
    "    testee_pam_std.append(testee.loc['std', 'pam'])\n",
    "    testee_phq4_std.append(testee.loc['std', 'phq4_score'])\n",
    "\n",
    "plt.figure(dpi=80,figsize=(10,10))\n",
    "plt.subplot(2,2,1)\n",
    "plt.hist(testee_pam_median,10)\n",
    "plt.title('pam median')\n",
    "plt.subplot(2,2,2)\n",
    "plt.hist(testee_phq4_median,10)\n",
    "plt.title('phq4 median')\n",
    "plt.subplot(2,2,3)\n",
    "plt.hist(testee_pam_std,10)\n",
    "plt.title('pam std')\n",
    "plt.subplot(2,2,4)\n",
    "plt.hist(testee_phq4_std,10)\n",
    "plt.title('phq4 std')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "做个时间序列展示："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = set(EMA['day'])\n",
    "a = sorted(a)\n",
    "a[-1]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "单个样本在采样时间上存在很大的差异，进一步考虑在所有样本中共有的采样时间"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = set(data['day'])\n",
    "for x in set(data['uid']):\n",
    "    testee = data[data['uid'] == x]\n",
    "    b = set(testee['day'])\n",
    "    a = set(list(a & b))\n",
    "    print(a)\n",
    "    print(b)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "看的出来，采样时间也非常散乱，没能取到所有采样时间的交集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "today_time =datetime.datetime.now()\n",
    "def date_time(starttime,endtime):\n",
    "    listx = []\n",
    "    starttime = datetime.datetime.strptime(starttime, '%Y-%m-%d')\n",
    "    endtime = datetime.datetime.strptime(endtime, '%Y-%m-%d')\n",
    "    while starttime<endtime:\n",
    "        select_time = starttime.strftime(\"%Y-%m-%d\")\n",
    "        select_time = str(select_time).replace('-','')\n",
    "        listx.append(int(select_time))\n",
    "        starttime+=datetime.timedelta(days=1)\n",
    "    return listx\n",
    "time_stamp = date_time(starttime='2017-09-07',endtime='2022-07-03')\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PAM区间为0~16，phq4区间为0~12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "time_stamp[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = list(range(len(time_stamp)))\n",
    "plt.figure(dpi=300,figsize=(50,10))\n",
    "plt.subplot(2,1,1)\n",
    "plt.title('pam')\n",
    "plt.xlim(0,len(time_stamp))\n",
    "plt.ylim(0,16)\n",
    "for y in range(len(EMA)):\n",
    "    time = int(EMA.loc[y, 'day'])\n",
    "    value = float(EMA.loc[y,'pam'])\n",
    "    plt.scatter([time_stamp.index(time)],[value],c='r',)\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "考虑用数据均值和标准差作图："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dicts = {}\n",
    "for x in range(len(EMA)):\n",
    "    if EMA.loc[x,'day'] in dicts.keys():\n",
    "        dicts[EMA.loc[x,'day']].append(EMA.loc[x,'pam'])\n",
    "    else:\n",
    "        dicts[EMA.loc[x,'day']] = []\n",
    "        dicts[EMA.loc[x,'day']].append(EMA.loc[x,'pam'])\n",
    "for x in dicts.keys():\n",
    "    para = dicts[x]\n",
    "    dicts[x] = [np.mean(para), np.std(para,ddof=1)]\n",
    "\n",
    "dict2 = {}\n",
    "for x in range(len(EMA)):\n",
    "    if EMA.loc[x,'day'] in dict2.keys():\n",
    "        dict2[EMA.loc[x,'day']].append(EMA.loc[x,'phq4_score'])\n",
    "    else:\n",
    "        dict2[EMA.loc[x,'day']] = []\n",
    "        dict2[EMA.loc[x,'day']].append(EMA.loc[x,'phq4_score'])\n",
    "for x in dict2.keys():\n",
    "    para = dict2[x]\n",
    "    dict2[x] = [np.mean(para), np.std(para,ddof=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dicts\n",
    "y = list(dicts.keys())\n",
    "y = sorted(y)\n",
    "plt.figure(dpi=300,figsize=(40,30))\n",
    "plt.subplot(2,1,1)\n",
    "plt.title('pam')\n",
    "plt.xlim(0,len(y))\n",
    "plt.ylim(0,20)\n",
    "for z in dicts.keys():\n",
    "    plt.scatter([y.index(z)],[dicts[z][0]],c='c')\n",
    "    plt.scatter([y.index(z)],[dicts[z][0] + dicts[z][1]],c='m')\n",
    "    plt.scatter([y.index(z)],[dicts[z][0] - dicts[z][1]],c='y')\n",
    "\n",
    "plt.subplot(2,1,2)\n",
    "plt.title('phq4')\n",
    "plt.xlim(0,len(y))\n",
    "plt.ylim(-5,15)\n",
    "for z in dict2.keys():\n",
    "    plt.scatter([y.index(z)],[dict2[z][0]],c='c')\n",
    "    plt.scatter([y.index(z)],[dict2[z][0] + dict2[z][1]],c='m')\n",
    "    plt.scatter([y.index(z)],[dict2[z][0] - dict2[z][1]],c='y')\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对所有的情绪量表参数都做以上处理以及可视化！"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "from importlib import reload\n",
    "sys.path.insert(0,r'G:\\main\\data\\Data Mining\\College Experience Study Dataset\\EMA')\n",
    "\n",
    "import emotion_visualization\n",
    "reload(emotion_visualization)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_list = list(data_main.columns)\n",
    "name_list.remove('uid')\n",
    "name_list.remove('day')\n",
    "for x in name_list:\n",
    "    emotion_visualization.emotion_visualization(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}