doppler_finder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Find images with doppler (they have DopplerFreq in their metadata)\n",
    "\n",
    "@author Pietro Moretto\n",
    "@version 0.1\n",
    "@date 2019/6/25\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import matplotlib.pyplot as plt\n",
    "import pydicom\n",
    "import cv2\n",
    "#from joblib import Parallel, delayed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ls \"/data3/wv2019/data/PLIC_MILANO_DICOM/PLICV4_DISCO4/GEMS_IMG/2013_JUL/05/AB090240/\"  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ROOT = \"/data3/wv2019/data/PLIC_MILANO_DICOM\"\n",
    "ROOT_OUT = \"/data3/wv2019/data/processed/PLIC_MILANO_DICOM\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define callbacks\n",
    "def person_names_callback(dataset, data_element):\n",
    "    if data_element.VR == \"PN\":\n",
    "        data_element.value = \"anonymous\"\n",
    "        \n",
    "def curves_callback(dataset, data_element):\n",
    "    if data_element.tag.group & 0xFF00 == 0x5000:\n",
    "        del dataset[data_element.tag]\n",
    "\n",
    "t2tag = 'PatientBirthDate'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### TODO\n",
    "# - consider also Doppler mode\n",
    "# - extend to the whole set of images\n",
    "# - check whether the anonymized DICOM file differs from the original (except for the removed sensitive data)\n",
    "# - parallelize code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_len = len(glob.glob(os.path.join(ROOT, \"*\", \"GEMS_IMG\", \"*\", \"*\", \"*\", \"*\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filepaths =[\"/data3/wv2019/data/PLIC_MILANO_DICOM/PLICV4_DISCO4/GEMS_IMG/2013_JUL/18/AC101435/D7IAB9D8\",\n",
    "            \"/data3/wv2019/data/PLIC_MILANO_DICOM/PLICV4_DISCO4/GEMS_IMG/2013_JUL/18/AC101435/D7IABAGE\",\n",
    "            \"/data3/wv2019/data/PLIC_MILANO_DICOM/PLICV4_DISCO4/GEMS_IMG/2013_JUL/18/AC101435/D7IABA5U\",\n",
    "            \"/data3/wv2019/data/PLIC_MILANO_DICOM/PLICV4_DISCO4/GEMS_IMG/2013_JUL/18/AC101435/D7IAB9D4\",\n",
    "            \"/data3/wv2019/data/PLIC_MILANO_DICOM/PLICV4_DISCO4/GEMS_IMG/2013_JUL/18/AC101435/D7IAB94K\",\n",
    "            \"/data3/wv2019/data/PLIC_MILANO_DICOM/PLICV4_DISCO4/GEMS_IMG/2013_JUL/18/AC101435/D7IAB9TM\",\n",
    "            \"/data3/wv2019/data/PLIC_MILANO_DICOM/PLICV4_DISCO4/GEMS_IMG/2013_JUL/18/AC101435/D7IAB9LE\"]\n",
    "\n",
    "for fn in filepaths:\n",
    "    ds = pydicom.read_file(fn)\n",
    "\n",
    "        # clear private data\n",
    "    ds.walk(person_names_callback)\n",
    "    ds.walk(curves_callback)\n",
    "        # address/sanitize patient IDs (remove spaces and non-alphanum characters)\n",
    "    ds.data_element('PatientID').value = re.sub(r'\\W+', '', ds.data_element('PatientID').value.strip().replace(\" \", \"_\"))\n",
    "        # type 2 tags\n",
    "    if t2tag in ds:\n",
    "        ds.data_element(t2tag).value = ''\n",
    "\n",
    "    patientID = ds.PatientID\n",
    "\n",
    "    dicomName = os.path.basename(fn).strip().replace(\" \", \"_\")\n",
    "\n",
    "    out_dir = os.path.join(ROOT_OUT, str(patientID))\n",
    "    os.makedirs(out_dir, exist_ok=True)\n",
    "    out_dicom = os.path.join(out_dir, f\"{dicomName}.dcm\")\n",
    "        #out_png = os.path.join(out_dir, f\"conv_{dicomName}.png\")\n",
    "    plt.imshow(ds.pixel_array)\n",
    "    plt.show()\n",
    "    \n",
    "    desc = str(ds)\n",
    "    for line in desc.split(\"\\n\"):\n",
    "        if \"doppler\" in line.lower():\n",
    "            print(line)\n",
    "    #print(str(ds))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "filepath2 = \"/data3/wv2019/data/PLIC_MILANO_DICOM/PLICV4_DISCO4/GEMS_IMG/2013_JUL/18/AC101435/D7IABA5U\"\n",
    "ds1 = pydicom.read_file(filepath2)\n",
    "\n",
    "    # clear private data\n",
    "ds1.walk(person_names_callback)\n",
    "ds1.walk(curves_callback)\n",
    "    # address/sanitize patient IDs (remove spaces and non-alphanum characters)\n",
    "ds1.data_element('PatientID').value = re.sub(r'\\W+', '', ds1.data_element('PatientID').value.strip().replace(\" \", \"_\"))\n",
    "    # type 2 tags\n",
    "if t2tag in ds1:\n",
    "    ds1.data_element(t2tag).value = ''\n",
    "\n",
    "patientID = ds1.PatientID\n",
    "dicomName = os.path.basename(filepath2).strip().replace(\" \", \"_\")\n",
    "\n",
    "out_dir = os.path.join(ROOT_OUT, str(patientID))\n",
    "os.makedirs(out_dir, exist_ok=True)\n",
    "out_dicom = os.path.join(out_dir, f\"{dicomName}.dcm\")\n",
    "    #out_png = os.path.join(out_dir, f\"conv_{dicomName}.png\")\n",
    "plt.imshow(ds1.pixel_array)\n",
    "plt.show()\n",
    "print(ds1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# directory tree:\n",
    "# ROOT\n",
    "# - PLIC<something>_DISCO<N>\n",
    "#   - GEMS_IMG\n",
    "#     - <YEAR>_<MONTH>\n",
    "#       - <NN>\n",
    "#         - <SOMEDIR>\n",
    "#           - DICOMs\n",
    "\n",
    "# organize DICOMs by patient ID\n",
    "# start with a subset of imgs\n",
    "for fn in glob.glob(os.path.join(ROOT, \"*\", \"GEMS_IMG\", \"*\", \"*\", \"*\", \"*\"), recursive=True)[:100]:\n",
    "    ds = pydicom.read_file(fn)\n",
    "\n",
    "    # clear private data\n",
    "    ds.walk(person_names_callback)\n",
    "    ds.walk(curves_callback)\n",
    "    # address/sanitize patient IDs (remove spaces and non-alphanum characters)\n",
    "    ds.data_element('PatientID').value = re.sub(r'\\W+', '', ds.data_element('PatientID').value.strip().replace(\" \", \"_\"))\n",
    "    # type 2 tags\n",
    "    if t2tag in ds:\n",
    "        ds.data_element(t2tag).value = ''\n",
    "\n",
    "    patientID = ds.PatientID\n",
    "    dicomName = os.path.basename(fn).strip().replace(\" \", \"_\")\n",
    "\n",
    "    out_dir = os.path.join(ROOT_OUT, str(patientID))\n",
    "    os.makedirs(out_dir, exist_ok=True)\n",
    "    out_dicom = os.path.join(out_dir, f\"{dicomName}.dcm\")\n",
    "    #out_png = os.path.join(out_dir, f\"conv_{dicomName}.png\")\n",
    "    plt.imshow(ds.pixel_array)\n",
    "    plt.show()\n",
    "    \n",
    "    print(fn)\n",
    "    \n",
    "    # write DICOM Standard compliant file\n",
    "    #if not os.path.isfile(out_dicom):\n",
    "    #    pydicom.filewriter.write_file(out_dicom, ds, write_like_original=False)\n",
    "    # write PNG for non-doppler images\n",
    "    # NOTE: not needed at first, let's keep also Doppler mode\n",
    "    #if len(ds.SequenceOfUltrasoundRegions) == 1 :\n",
    "    #    if not os.path.isfile(out_png):\n",
    "    #        cv2.imwrite(out_png, ds.pixel_array)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}