10_process_simulated_tasic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ddbdebb6-7315-41f9-a67c-6fd7844940cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scanpy as sc\n",
    "import anndata\n",
    "import openTSNE\n",
    "from openTSNE.initialization import rescale as rescale_pca\n",
    "from sklearn.decomposition import PCA\n",
    "sc.settings.verbosity = 2\n",
    "from readcount_tools import get_tag, readcount_pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4071c2d7-f4af-4d76-aaa3-14a00718e0d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/anndata/_core/anndata.py:1830: UserWarning: Variable names are not unique. To make them unique, call `.var_names_make_unique`.\n",
      "  utils.warn_names_duplicates(\"var\")\n",
      "/usr/local/lib/python3.8/dist-packages/anndata/_core/anndata.py:1830: UserWarning: Variable names are not unique. To make them unique, call `.var_names_make_unique`.\n",
      "  utils.warn_names_duplicates(\"var\")\n"
     ]
    }
   ],
   "source": [
    "path = 'data/tasic/simulations/adata_sim_thetaUMI_100_GeomZ'\n",
    "adata_sim = anndata.read_h5ad(path+'.h5ad')\n",
    "path_only_marker_genes_DE = 'data/tasic/simulations/adata_sim_only_marker_genes_DE_thetaUMI_100_GeomZ'\n",
    "adata_sim_only_marker_genes_DE = anndata.read_h5ad(path_only_marker_genes_DE+'.h5ad')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "038d0893-ba02-4b0c-b24c-68def3f087fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "theta_umi = 100\n",
    "n_hvgs = 3000\n",
    "n_comps = 1000\n",
    "alphas = [1,50,199]\n",
    "adatas = [adata_sim, adata_sim_only_marker_genes_DE]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a9bcdad6-2198-41de-a302-64eb51b7e135",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "filtered out 12124 genes that are detected in less than 5 cells\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/anndata/_core/anndata.py:1830: UserWarning: Variable names are not unique. To make them unique, call `.var_names_make_unique`.\n",
      "  utils.warn_names_duplicates(\"var\")\n",
      "tcmalloc: large alloc 2920775680 bytes == 0x22eef4000 @ \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pr_theta100_alpha1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tcmalloc: large alloc 2920775680 bytes == 0x2dd06c000 @ \n",
      "tcmalloc: large alloc 2920775680 bytes == 0x38b9e4000 @ \n",
      "tcmalloc: large alloc 2920775680 bytes == 0x439b5c000 @ \n",
      "tcmalloc: large alloc 2920775680 bytes == 0x4e7cd4000 @ \n",
      "tcmalloc: large alloc 2920775680 bytes == 0x4e7cd4000 @ \n",
      "tcmalloc: large alloc 2920775680 bytes == 0x439b5c000 @ \n",
      "tcmalloc: large alloc 2920775680 bytes == 0x22eef4000 @ \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pr_theta100_alpha1.0_afterHVG\n",
      "--------------------------------------------------------------------------------\n",
      "TSNE(n_jobs=38, random_state=42, verbose=True)\n",
      "--------------------------------------------------------------------------------\n",
      "===> Finding 90 nearest neighbors using Annoy approximate search using euclidean distance...\n",
      "   --> Time elapsed: 6.69 seconds\n",
      "===> Calculating affinity matrix...\n",
      "   --> Time elapsed: 0.19 seconds\n",
      "===> Calculating PCA-based initialization...\n",
      "   --> Time elapsed: 0.48 seconds\n",
      "===> Running optimization with exaggeration=12.00, lr=1985.17 for 250 iterations...\n",
      "Iteration   50, KL divergence 5.7719, 50 iterations in 1.6141 sec\n",
      "Iteration  100, KL divergence 4.8098, 50 iterations in 1.7537 sec\n",
      "Iteration  150, KL divergence 4.5192, 50 iterations in 1.7109 sec\n",
      "Iteration  200, KL divergence 4.3580, 50 iterations in 1.7292 sec\n",
      "Iteration  250, KL divergence 4.2699, 50 iterations in 1.6265 sec\n",
      "   --> Time elapsed: 8.44 seconds\n",
      "===> Running optimization with exaggeration=1.00, lr=1985.17 for 500 iterations...\n",
      "Iteration   50, KL divergence 3.1270, 50 iterations in 1.4984 sec\n",
      "Iteration  100, KL divergence 2.7664, 50 iterations in 1.6778 sec\n",
      "Iteration  150, KL divergence 2.6197, 50 iterations in 1.6208 sec\n",
      "Iteration  200, KL divergence 2.5438, 50 iterations in 2.0810 sec\n",
      "Iteration  250, KL divergence 2.5037, 50 iterations in 3.0172 sec\n",
      "Iteration  300, KL divergence 2.4771, 50 iterations in 3.0070 sec\n",
      "Iteration  350, KL divergence 2.4608, 50 iterations in 3.7391 sec\n",
      "Iteration  400, KL divergence 2.4470, 50 iterations in 4.3596 sec\n",
      "Iteration  450, KL divergence 2.4362, 50 iterations in 4.5543 sec\n",
      "Iteration  500, KL divergence 2.4285, 50 iterations in 4.4424 sec\n",
      "   --> Time elapsed: 30.00 seconds\n",
      "filtered out 12200 genes that are detected in less than 5 cells\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/anndata/_core/anndata.py:1830: UserWarning: Variable names are not unique. To make them unique, call `.var_names_make_unique`.\n",
      "  utils.warn_names_duplicates(\"var\")\n",
      "tcmalloc: large alloc 2913533952 bytes == 0x439b5c000 @ \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pr_theta100_alpha1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/anndata/_core/anndata.py:1830: UserWarning: Variable names are not unique. To make them unique, call `.var_names_make_unique`.\n",
      "  utils.warn_names_duplicates(\"var\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pr_theta100_alpha1.0_afterHVG\n",
      "--------------------------------------------------------------------------------\n",
      "TSNE(n_jobs=38, random_state=42, verbose=True)\n",
      "--------------------------------------------------------------------------------\n",
      "===> Finding 90 nearest neighbors using Annoy approximate search using euclidean distance...\n",
      "   --> Time elapsed: 8.48 seconds\n",
      "===> Calculating affinity matrix...\n",
      "   --> Time elapsed: 0.26 seconds\n",
      "===> Calculating PCA-based initialization...\n",
      "   --> Time elapsed: 0.56 seconds\n",
      "===> Running optimization with exaggeration=12.00, lr=1985.17 for 250 iterations...\n",
      "Iteration   50, KL divergence 6.7454, 50 iterations in 1.5842 sec\n",
      "Iteration  100, KL divergence 6.6367, 50 iterations in 1.8444 sec\n",
      "Iteration  150, KL divergence 6.6601, 50 iterations in 1.5287 sec\n",
      "Iteration  200, KL divergence 6.7405, 50 iterations in 1.5460 sec\n",
      "Iteration  250, KL divergence 6.6861, 50 iterations in 1.6035 sec\n",
      "   --> Time elapsed: 8.11 seconds\n",
      "===> Running optimization with exaggeration=1.00, lr=1985.17 for 500 iterations...\n",
      "Iteration   50, KL divergence 5.3491, 50 iterations in 1.5099 sec\n",
      "Iteration  100, KL divergence 5.2262, 50 iterations in 1.5089 sec\n",
      "Iteration  150, KL divergence 5.2099, 50 iterations in 1.5328 sec\n",
      "Iteration  200, KL divergence 5.1632, 50 iterations in 1.5627 sec\n",
      "Iteration  250, KL divergence 5.1539, 50 iterations in 1.5571 sec\n",
      "Iteration  300, KL divergence 5.1826, 50 iterations in 1.5338 sec\n",
      "Iteration  350, KL divergence 5.1508, 50 iterations in 1.5250 sec\n",
      "Iteration  400, KL divergence 5.1488, 50 iterations in 1.7321 sec\n",
      "Iteration  450, KL divergence 5.1740, 50 iterations in 1.6481 sec\n",
      "Iteration  500, KL divergence 5.1835, 50 iterations in 1.7786 sec\n",
      "   --> Time elapsed: 15.89 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/anndata/_core/anndata.py:1830: UserWarning: Variable names are not unique. To make them unique, call `.var_names_make_unique`.\n",
      "  utils.warn_names_duplicates(\"var\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pr_theta100_alpha50.0\n",
      "pr_theta100_alpha50.0_afterHVG\n",
      "--------------------------------------------------------------------------------\n",
      "TSNE(n_jobs=38, random_state=42, verbose=True)\n",
      "--------------------------------------------------------------------------------\n",
      "===> Finding 90 nearest neighbors using Annoy approximate search using euclidean distance...\n",
      "   --> Time elapsed: 7.05 seconds\n",
      "===> Calculating affinity matrix...\n",
      "   --> Time elapsed: 0.18 seconds\n",
      "===> Calculating PCA-based initialization...\n",
      "   --> Time elapsed: 0.53 seconds\n",
      "===> Running optimization with exaggeration=12.00, lr=1985.17 for 250 iterations...\n",
      "Iteration   50, KL divergence 5.5140, 50 iterations in 1.4108 sec\n",
      "Iteration  100, KL divergence 4.2558, 50 iterations in 1.6000 sec\n",
      "Iteration  150, KL divergence 3.8645, 50 iterations in 1.7418 sec\n",
      "Iteration  200, KL divergence 3.6439, 50 iterations in 1.4766 sec\n",
      "Iteration  250, KL divergence 3.5034, 50 iterations in 1.2979 sec\n",
      "   --> Time elapsed: 7.53 seconds\n",
      "===> Running optimization with exaggeration=1.00, lr=1985.17 for 500 iterations...\n",
      "Iteration   50, KL divergence 2.7388, 50 iterations in 1.3047 sec\n",
      "Iteration  100, KL divergence 2.3147, 50 iterations in 1.2554 sec\n",
      "Iteration  150, KL divergence 2.1128, 50 iterations in 1.8623 sec\n",
      "Iteration  200, KL divergence 2.0034, 50 iterations in 3.1324 sec\n",
      "Iteration  250, KL divergence 1.9400, 50 iterations in 3.4603 sec\n",
      "Iteration  300, KL divergence 1.8983, 50 iterations in 4.1856 sec\n",
      "Iteration  350, KL divergence 1.8700, 50 iterations in 4.6611 sec\n",
      "Iteration  400, KL divergence 1.8474, 50 iterations in 5.8678 sec\n",
      "Iteration  450, KL divergence 1.8275, 50 iterations in 6.8222 sec\n",
      "Iteration  500, KL divergence 1.8108, 50 iterations in 7.7638 sec\n",
      "   --> Time elapsed: 40.32 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/anndata/_core/anndata.py:1830: UserWarning: Variable names are not unique. To make them unique, call `.var_names_make_unique`.\n",
      "  utils.warn_names_duplicates(\"var\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pr_theta100_alpha50.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/anndata/_core/anndata.py:1830: UserWarning: Variable names are not unique. To make them unique, call `.var_names_make_unique`.\n",
      "  utils.warn_names_duplicates(\"var\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pr_theta100_alpha50.0_afterHVG\n",
      "--------------------------------------------------------------------------------\n",
      "TSNE(n_jobs=38, random_state=42, verbose=True)\n",
      "--------------------------------------------------------------------------------\n",
      "===> Finding 90 nearest neighbors using Annoy approximate search using euclidean distance...\n",
      "   --> Time elapsed: 7.42 seconds\n",
      "===> Calculating affinity matrix...\n",
      "   --> Time elapsed: 0.20 seconds\n",
      "===> Calculating PCA-based initialization...\n",
      "   --> Time elapsed: 0.65 seconds\n",
      "===> Running optimization with exaggeration=12.00, lr=1985.17 for 250 iterations...\n",
      "Iteration   50, KL divergence 6.3205, 50 iterations in 1.6200 sec\n",
      "Iteration  100, KL divergence 6.1152, 50 iterations in 1.7312 sec\n",
      "Iteration  150, KL divergence 6.0870, 50 iterations in 1.4613 sec\n",
      "Iteration  200, KL divergence 6.0233, 50 iterations in 1.6383 sec\n",
      "Iteration  250, KL divergence 6.0007, 50 iterations in 1.6976 sec\n",
      "   --> Time elapsed: 8.15 seconds\n",
      "===> Running optimization with exaggeration=1.00, lr=1985.17 for 500 iterations...\n",
      "Iteration   50, KL divergence 4.4068, 50 iterations in 1.6780 sec\n",
      "Iteration  100, KL divergence 4.1653, 50 iterations in 1.5881 sec\n",
      "Iteration  150, KL divergence 4.0690, 50 iterations in 1.6254 sec\n",
      "Iteration  200, KL divergence 4.0138, 50 iterations in 1.7634 sec\n",
      "Iteration  250, KL divergence 3.9815, 50 iterations in 2.2699 sec\n",
      "Iteration  300, KL divergence 3.9592, 50 iterations in 2.6683 sec\n",
      "Iteration  350, KL divergence 3.9448, 50 iterations in 2.7302 sec\n",
      "Iteration  400, KL divergence 3.9325, 50 iterations in 3.0386 sec\n",
      "Iteration  450, KL divergence 3.9235, 50 iterations in 3.1905 sec\n",
      "Iteration  500, KL divergence 3.9153, 50 iterations in 3.4846 sec\n",
      "   --> Time elapsed: 24.04 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/anndata/_core/anndata.py:1830: UserWarning: Variable names are not unique. To make them unique, call `.var_names_make_unique`.\n",
      "  utils.warn_names_duplicates(\"var\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pr_theta100_alpha199.0\n",
      "pr_theta100_alpha199.0_afterHVG\n",
      "--------------------------------------------------------------------------------\n",
      "TSNE(n_jobs=38, random_state=42, verbose=True)\n",
      "--------------------------------------------------------------------------------\n",
      "===> Finding 90 nearest neighbors using Annoy approximate search using euclidean distance...\n",
      "   --> Time elapsed: 6.48 seconds\n",
      "===> Calculating affinity matrix...\n",
      "   --> Time elapsed: 0.17 seconds\n",
      "===> Calculating PCA-based initialization...\n",
      "   --> Time elapsed: 0.53 seconds\n",
      "===> Running optimization with exaggeration=12.00, lr=1985.17 for 250 iterations...\n",
      "Iteration   50, KL divergence 5.6379, 50 iterations in 1.3402 sec\n",
      "Iteration  100, KL divergence 4.2452, 50 iterations in 1.4593 sec\n",
      "Iteration  150, KL divergence 3.8193, 50 iterations in 1.7282 sec\n",
      "Iteration  200, KL divergence 3.6030, 50 iterations in 1.5404 sec\n",
      "Iteration  250, KL divergence 3.4588, 50 iterations in 1.6651 sec\n",
      "   --> Time elapsed: 7.73 seconds\n",
      "===> Running optimization with exaggeration=1.00, lr=1985.17 for 500 iterations...\n",
      "Iteration   50, KL divergence 2.6926, 50 iterations in 1.3907 sec\n",
      "Iteration  100, KL divergence 2.2558, 50 iterations in 1.6048 sec\n",
      "Iteration  150, KL divergence 2.0367, 50 iterations in 1.9848 sec\n",
      "Iteration  200, KL divergence 1.9183, 50 iterations in 2.9041 sec\n",
      "Iteration  250, KL divergence 1.8484, 50 iterations in 3.4553 sec\n",
      "Iteration  300, KL divergence 1.8029, 50 iterations in 4.8830 sec\n",
      "Iteration  350, KL divergence 1.7713, 50 iterations in 8.8511 sec\n",
      "Iteration  400, KL divergence 1.7473, 50 iterations in 10.8025 sec\n",
      "Iteration  450, KL divergence 1.7275, 50 iterations in 11.0360 sec\n",
      "Iteration  500, KL divergence 1.7112, 50 iterations in 12.8367 sec\n",
      "   --> Time elapsed: 59.76 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/anndata/_core/anndata.py:1830: UserWarning: Variable names are not unique. To make them unique, call `.var_names_make_unique`.\n",
      "  utils.warn_names_duplicates(\"var\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pr_theta100_alpha199.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/anndata/_core/anndata.py:1830: UserWarning: Variable names are not unique. To make them unique, call `.var_names_make_unique`.\n",
      "  utils.warn_names_duplicates(\"var\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pr_theta100_alpha199.0_afterHVG\n",
      "--------------------------------------------------------------------------------\n",
      "TSNE(n_jobs=38, random_state=42, verbose=True)\n",
      "--------------------------------------------------------------------------------\n",
      "===> Finding 90 nearest neighbors using Annoy approximate search using euclidean distance...\n",
      "   --> Time elapsed: 7.57 seconds\n",
      "===> Calculating affinity matrix...\n",
      "   --> Time elapsed: 0.23 seconds\n",
      "===> Calculating PCA-based initialization...\n",
      "   --> Time elapsed: 0.49 seconds\n",
      "===> Running optimization with exaggeration=12.00, lr=1985.17 for 250 iterations...\n",
      "Iteration   50, KL divergence 6.2091, 50 iterations in 1.6845 sec\n",
      "Iteration  100, KL divergence 5.9686, 50 iterations in 1.5772 sec\n",
      "Iteration  150, KL divergence 5.8932, 50 iterations in 1.5383 sec\n",
      "Iteration  200, KL divergence 5.8192, 50 iterations in 1.5880 sec\n",
      "Iteration  250, KL divergence 5.8379, 50 iterations in 1.6605 sec\n",
      "   --> Time elapsed: 8.05 seconds\n",
      "===> Running optimization with exaggeration=1.00, lr=1985.17 for 500 iterations...\n",
      "Iteration   50, KL divergence 4.1810, 50 iterations in 1.7171 sec\n",
      "Iteration  100, KL divergence 3.8760, 50 iterations in 1.5234 sec\n",
      "Iteration  150, KL divergence 3.7329, 50 iterations in 1.4186 sec\n",
      "Iteration  200, KL divergence 3.6467, 50 iterations in 1.6202 sec\n",
      "Iteration  250, KL divergence 3.5889, 50 iterations in 2.5226 sec\n",
      "Iteration  300, KL divergence 3.5466, 50 iterations in 2.9095 sec\n",
      "Iteration  350, KL divergence 3.5140, 50 iterations in 3.4653 sec\n",
      "Iteration  400, KL divergence 3.4880, 50 iterations in 4.4669 sec\n",
      "Iteration  450, KL divergence 3.4681, 50 iterations in 4.3761 sec\n",
      "Iteration  500, KL divergence 3.4519, 50 iterations in 5.5018 sec\n",
      "   --> Time elapsed: 29.53 seconds\n"
     ]
    }
   ],
   "source": [
    "for alpha in alphas:\n",
    "    [readcount_pipeline(ad,alpha=alpha,theta=theta_umi,n_hvgs=n_hvgs,n_comps=n_comps) for ad in adatas]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3ca881a1-9e2a-4684-b4e0-5517083314fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(23822, 30652)\n",
      "(23822, 30576)\n"
     ]
    }
   ],
   "source": [
    "print(adata_sim.shape)\n",
    "print(adata_sim_only_marker_genes_DE.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b44cf65a-8778-4785-b4c2-f777229d23ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_sim.write_h5ad(path+'_processed.h5ad')\n",
    "adata_sim_only_marker_genes_DE.write_h5ad(path_only_marker_genes_DE+'_processed.h5ad')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}