file_based_function_execution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f444b8be",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "31424aca",
   "metadata": {},
   "outputs": [],
   "source": [
    "import hashlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1f7e5af3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5778e7ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "import cloudpickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "21e192ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5a24842f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_hash(binary):\n",
    "    binary_no_ipykernel = re.sub(b\"(?<=/ipykernel_)(.*)(?=/)\", b\"\", binary)\n",
    "    return str(hashlib.md5(binary_no_ipykernel).hexdigest())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "02321a2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def serialize_funct(fn, *args, **kwargs):\n",
    "    binary = cloudpickle.dumps({\"fn\": fn, \"args\": args, \"kwargs\": kwargs})\n",
    "    return {fn.__name__ + get_hash(binary=binary): binary}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8d10ae15",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_file_name(name, state):\n",
    "    return name + \".\" + state + \".pl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7deaf07f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_to_file(funct_dict, state):\n",
    "    file_name_lst = []\n",
    "    for k, v in funct_dict.items():\n",
    "        file_name = get_file_name(name=k, state=state)\n",
    "        file_name_lst.append(file_name)\n",
    "        with open(file_name, \"wb\") as f:\n",
    "            f.write(v)\n",
    "    return file_name_lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "75ccc65f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_from_file(file_name):\n",
    "    name = file_name.split(\".\")[0]\n",
    "    with open(file_name, \"rb\") as f:\n",
    "        return {name: f.read()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fbddad52",
   "metadata": {},
   "outputs": [],
   "source": [
    "def deserialize(funct_dict):\n",
    "    return {k: cloudpickle.loads(v) for k, v in funct_dict.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "13ba18b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_funct(apply_dict):\n",
    "    return {k: v[\"fn\"].__call__(*v[\"args\"], **v[\"kwargs\"]) for k, v in apply_dict.items()} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "36d9346f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def serialize_result(result_dict):\n",
    "    return {k: cloudpickle.dumps(v) for k, v in result_dict.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "348e4f2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def execute_function(fn, *args, **kwargs):\n",
    "    funct_dict = serialize_funct(fn, *args, **kwargs)\n",
    "    file_lst = os.listdir()\n",
    "    file_dict = {k: [f for f in file_lst if k in f] for k in funct_dict.keys()}\n",
    "    for k, v in file_dict.items():\n",
    "        if len(v) == 2: \n",
    "            file_name_result = [f for f in v if f.endswith(\"out.pl\")][0]   \n",
    "        else:\n",
    "            if len(v) == 0:\n",
    "                file_name_funct = write_to_file(\n",
    "                    funct_dict=serialize_funct(fn, *args, **kwargs), \n",
    "                    state=\"in\"\n",
    "                )[0]\n",
    "            else:\n",
    "                file_name_funct = v[0]\n",
    "            funct_dict = read_from_file(file_name=file_name_funct)\n",
    "            apply_dict = deserialize(funct_dict=funct_dict)\n",
    "            result_dict = apply_funct(apply_dict=apply_dict)\n",
    "            write_to_file(funct_dict=serialize_result(result_dict=result_dict), state=\"out\")\n",
    "            file_name_result = file_name_funct.replace(\"in.pl\", \"out.pl\")\n",
    "        return list(deserialize(funct_dict=read_from_file(file_name=file_name_result)).values())[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1301e2de",
   "metadata": {},
   "source": [
    "# Write function to file "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "27f4ebad",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_funct(a, b):\n",
    "    time.sleep(5)\n",
    "    return a+b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "31630552",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "execute_function(test_funct, 1, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c522fd96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['store_function_v4.ipynb',\n",
       " 'store_function_v6.ipynb',\n",
       " 'store_function_v2.ipynb',\n",
       " 'store_function_v5.ipynb',\n",
       " 'store_function_v1.ipynb',\n",
       " 'store_function_v3.ipynb',\n",
       " 'test_funct93f7c993ad3e89a2fb17716ef985a8a6.in.pl',\n",
       " 'test_funct93f7c993ad3e89a2fb17716ef985a8a6.out.pl',\n",
       " '.ipynb_checkpoints']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir()  # test_funct93f7c993ad3e89a2fb17716ef985a8a6.out.pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "828d430f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "execute_function(test_funct, 1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2d9c85cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['store_function_v4.ipynb',\n",
       " 'test_funct9af832e861123b84e12ff5dea30068af.out.pl',\n",
       " 'store_function_v6.ipynb',\n",
       " 'store_function_v2.ipynb',\n",
       " 'store_function_v5.ipynb',\n",
       " 'store_function_v1.ipynb',\n",
       " 'store_function_v3.ipynb',\n",
       " 'test_funct93f7c993ad3e89a2fb17716ef985a8a6.in.pl',\n",
       " 'test_funct93f7c993ad3e89a2fb17716ef985a8a6.out.pl',\n",
       " 'test_funct9af832e861123b84e12ff5dea30068af.in.pl',\n",
       " '.ipynb_checkpoints']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "34955b18",
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in os.listdir():\n",
    "    if f.endswith(\"out.pl\"):\n",
    "        os.remove(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d81478b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['store_function_v4.ipynb',\n",
       " 'store_function_v6.ipynb',\n",
       " 'store_function_v2.ipynb',\n",
       " 'store_function_v5.ipynb',\n",
       " 'store_function_v1.ipynb',\n",
       " 'store_function_v3.ipynb',\n",
       " 'test_funct93f7c993ad3e89a2fb17716ef985a8a6.in.pl',\n",
       " 'test_funct9af832e861123b84e12ff5dea30068af.in.pl',\n",
       " '.ipynb_checkpoints']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1c47a5a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "execute_function(test_funct, 1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "fbcefd6b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['store_function_v4.ipynb',\n",
       " 'test_funct9af832e861123b84e12ff5dea30068af.out.pl',\n",
       " 'store_function_v6.ipynb',\n",
       " 'store_function_v2.ipynb',\n",
       " 'store_function_v5.ipynb',\n",
       " 'store_function_v1.ipynb',\n",
       " 'store_function_v3.ipynb',\n",
       " 'test_funct93f7c993ad3e89a2fb17716ef985a8a6.in.pl',\n",
       " 'test_funct9af832e861123b84e12ff5dea30068af.in.pl',\n",
       " '.ipynb_checkpoints']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f3ddbcbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in os.listdir():\n",
    "    if f.endswith(\".pl\"):\n",
    "        os.remove(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1e2626b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['store_function_v4.ipynb',\n",
       " 'store_function_v6.ipynb',\n",
       " 'store_function_v2.ipynb',\n",
       " 'store_function_v5.ipynb',\n",
       " 'store_function_v1.ipynb',\n",
       " 'store_function_v3.ipynb',\n",
       " '.ipynb_checkpoints']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2715e627",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75377253",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}