Merge pull request #270 from creatermaker/main

image-tiling component improved

component-library/transform/image-tiling.cwl

@@ -0,0 +1,57 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+baseCommand: "claimed"
+
+inputs:
+  component:
+    type: string
+    default: docker.io/mdorzweiler/claimed-image-tiling:0.1
+    inputBinding:
+      position: 1
+      prefix: --component
+  log_level:
+    type: string
+    default: "INFO"
+    inputBinding:
+      position: 2
+      prefix: --log_level
+  directory_path:
+    type: string
+    default: None
+    inputBinding:
+      position: 3
+      prefix: --directory_path
+  pattern:
+    type: string
+    default: None
+    inputBinding:
+      position: 4
+      prefix: --pattern
+  tile_size_x:
+    type: string
+    default: None
+    inputBinding:
+      position: 5
+      prefix: --tile_size_x
+  tile_size_y:
+    type: string
+    default: None
+    inputBinding:
+      position: 6
+      prefix: --tile_size_y
+  stride_x:
+    type: string
+    default: None
+    inputBinding:
+      position: 7
+      prefix: --stride_x
+  stride_y:
+    type: string
+    default: None
+    inputBinding:
+      position: 8
+      prefix: --stride_y
+
+
+outputs: []

component-library/transform/image-tiling.ipynb

@@ -0,0 +1,191 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "2e8a0fac-adbd-4428-90ea-869aee2b95bf",
+   "metadata": {},
+   "source": [
+    "## Xview Dataset tiling\n",
+    "This is a component designed to tile the dataset provided by xview. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c0421aaf-4da4-4a66-b626-e8962315afe8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install Pillow"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0b7ad533-dabc-488e-bf65-59a54a85f3f9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "import os\n",
+    "import glob \n",
+    "import logging\n",
+    "from PIL import Image\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "61cf8267-ecba-4732-bd76-094bb604f0e8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def to_int(value):\n",
+    "    return int(value)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c8df047b-6167-43f1-9a31-d836580180ac",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "#directory_path is the path to the folder with the unzipped .tif images from xview dataset \n",
+    "directory_path = os.environ.get(\"directory_path\")\n",
+    "\n",
+    "#pattern is the search pattern a user can pass for the program to extract the files from \n",
+    "pattern = os.environ.get(\"pattern\")\n",
+    "\n",
+    "#Each image is cropped using a rectangular window with edge_length tile_size_x and tile_size_y which has to be given in number of pixels \n",
+    "tile_size_x = to_int(os.environ.get(\"tile_size_x\"))\n",
+    "tile_size_y = to_int(os.environ.get(\"tile_size_y\"))\n",
+    "\n",
+    "#stride_x is the length in pixels the sliding window is moved to the right after each step\n",
+    "#stride_y is the length in pixels the sliding window is moved down after completing a row\n",
+    "#For tumbling window stride_x must equal tile_size_x and stride_y must equal tile_size_y\n",
+    "stride_x = to_int(os.environ.get(\"stride_x\"))\n",
+    "stride_y = to_int(os.environ.get(\"stride_y\"))\n",
+    "\n",
+    "# log level\n",
+    "log_level = os.environ.get('log_level', 'INFO')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8ed74228-937d-4d5d-b6d9-97026af3eb50",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "def clear_directory(directory):\n",
+    "    logging.info(f'start clearing directory {directory}')\n",
+    "    for item in os.listdir(directory):\n",
+    "            item_path = os.path.join(directory, item)\n",
+    "            # Check if the item is a file\n",
+    "            if os.path.isfile(item_path):\n",
+    "                # If it's a file, remove it\n",
+    "                os.remove(item_path)\n",
+    "            # If it's a directory, recursively clear it\n",
+    "            elif os.path.isdir(item_path):\n",
+    "                clear_directory(item_path)\n",
+    "                os.rmdir(item_path)\n",
+    "    logging.ingo(f'directory {directory} cleared successfully')\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d6e89f94-490a-4c1a-80cd-f2ab30adb374",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "def sliding_window(directory_path, destination, tile_size_x, tile_size_y, stride_x, stride_y):\n",
+    "    \n",
+    "    if os.path.isdir(destination):\n",
+    "        logging.info(f'destination folder {destination} already exists')\n",
+    "        clear_directory(destination)\n",
+    "    else: \n",
+    "        logging.info(f'destination folder {destination} does not exist yet, creating folder')\n",
+    "        os.makedirs(destination)\n",
+    "        logging.info(f'destination folder {destination} created successfully')\n",
+    "\n",
+    "    items = glob.glob(pattern, recursive=True)\n",
+    "    \n",
+    "    for item in items:\n",
+    "\n",
+    "        logging.info(f'grabbed item {item}, starting the tiling process')\n",
+    "        \n",
+    "        item_path = os.path.join(directory_path, item)\n",
+    "        clipped_item = os.path.join(destination, item)\n",
+    "        os.makedirs(clipped_item)\n",
+    "        \n",
+    "        image = Image.open(item_path)\n",
+    "        logging.info(f'{item} loaded successfully')\n",
+    "        width, height = image.size\n",
+    "        \n",
+    "        x_range = [0]\n",
+    "        while(x_range[-1] + stride_x + tile_size_x < width):\n",
+    "            x_range += [x_range[-1] + stride_x]\n",
+    "        \n",
+    "        y_range = [0]\n",
+    "        while(y_range[-1] + stride_y + tile_size_y < height):\n",
+    "            y_range += [y_range[-1] + stride_y]\n",
+    "\n",
+    "        logging.info(f'successfully partitioned the height and width of {item}')\n",
+    "        \n",
+    "        counter = 0\n",
+    "        for x in x_range:\n",
+    "            for y in y_range:\n",
+    "                cropped = image.crop((x,y, x+tile_size_x, y+tile_size_y))\n",
+    "\n",
+    "                logging.info(f'successfully extracted a tile from {item} at x={x} and y={y}')\n",
+    "                \n",
+    "                file_name = \"{}\".format(y)\n",
+    "                file_name += \"_{}\".format(x) + \".tif\"\n",
+    "                path = os.path.join(clipped_item, file_name)\n",
+    "                \n",
+    "                cropped.save(path)\n",
+    "\n",
+    "                logging.info(f'successfully saved tile from {item} at x={x} and y={y} in {path}')\n",
+    "                \n",
+    "        logging.info(f'Successfully finished tiling {item}')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ce60bf47-7235-43a1-a4f7-5b81080d617d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sliding_window(directory_path, destination, tile_size_x, tile_size_y, stride_x, stride_y)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

component-library/transform/image-tiling.job.yaml

@@ -0,0 +1,30 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: image-tiling
+spec:
+  template:
+    spec:
+      containers:
+      - name: image-tiling
+        image: docker.io/mdorzweiler/claimed-image-tiling:0.1
+        workingDir: /opt/app-root/src/
+        command: ["/opt/app-root/bin/ipython","/Users/maxdorzweiler/Desktop/ibm/clipping/claimed_image-tiling.ipynb"]
+        env:
+        - name: log_level
+          value: value_of_log_level
+        - name: directory_path
+          value: value_of_directory_path
+        - name: pattern
+          value: value_of_pattern
+        - name: tile_size_x
+          value: value_of_tile_size_x
+        - name: tile_size_y
+          value: value_of_tile_size_y
+        - name: stride_x
+          value: value_of_stride_x
+        - name: stride_y
+          value: value_of_stride_y
+      restartPolicy: OnFailure
+      imagePullSecrets:
+        - name: image_pull_secret

component-library/transform/image-tiling.yaml

@@ -0,0 +1,31 @@
+name: image-tiling
+description: "## Xview Dataset tiling  – CLAIMED V0.1"
+
+inputs:
+- {name: log_level, type: String, description: "update log level", default: "INFO"}
+- {name: directory_path, type: String, description: "directory_path is the path to the folder with the unzipped .tif images from xview dataset"}
+- {name: pattern, type: String, description: "pattern is the search pattern a user can pass for the program to extract the files from"}
+- {name: tile_size_x, type: String, description: "Each image is cropped using a rectangular window with edge_length tile_size_x and tile_size_y which has to be given in number of pixels"}
+- {name: tile_size_y, type: String, description: ""}
+- {name: stride_x, type: String, description: "For tumbling window stride_x must equal tile_size_x and stride_y must equal tile_size_y"}
+- {name: stride_y, type: String, description: ""}
+
+
+outputs:
+
+
+implementation:
+    container:
+        image: docker.io/mdorzweiler/claimed-image-tiling:0.1
+        command:
+        - sh
+        - -ec
+        - |
+          ipython .//Users/maxdorzweiler/Desktop/ibm/clipping/claimed_image-tiling.ipynb log_level="${0}" directory_path="${1}" pattern="${2}" tile_size_x="${3}" tile_size_y="${4}" stride_x="${5}" stride_y="${6}" 
+        - {inputValue: log_level}
+        - {inputValue: directory_path}
+        - {inputValue: pattern}
+        - {inputValue: tile_size_x}
+        - {inputValue: tile_size_y}
+        - {inputValue: stride_x}
+        - {inputValue: stride_y}