add training config and bash

bash/convert_to_tfrecords.sh

@@ -0,0 +1,2 @@
+export PYTHONPATH="${PYTHONPATH}:./"
+python Demic/test/test_convert_to_tfrecords.py config/write_tfrecords_fb.txt

bash/evaluate.sh

@@ -0,0 +1,35 @@
+#$ -P gpu
+#$ -l gpu=1
+#$ -l gpu_pascal=1
+#$ -l h_rt=10:0:0
+#$ -l tmem=10.0G
+##$ -l h=tesla5
+#$ -N pnet_test
+#$ -S /bin/bash
+#$ -wd /home/guotwang/tf_project/test_aug/brats/bash
+#!/bin/bash
+# The lines above are resource requests. This script has requested 1 Titan X (Pascal) GPU for 24 hours, and 11.5 GB of memory to be started with the BASH Shell.
+# More information about resource requests can be found at http://hpc.cs.ucl.ac.uk/job_submission_sge/
+
+# This line ensures that you only use the 1 GPU requested.
+nvidia-smi
+export CUDA_VISIBLE_DEVICES=$(( `nvidia-smi | grep " / .....MiB"|grep -n " ...MiB / [0-9]....MiB"|cut -d : -f 1|head -n 1` - 1 ))
+echo $CUDA_VISIBLE_DEVICES
+if [ $CUDA_VISIBLE_DEVICES -lt 0 ];then
+exit 1
+fi
+# These lines runs your NiftyNet task with the correct library paths for tensorflow
+TF_LD_LIBRARY_PATH=/share/apps/libc6_2.17/lib/x86_64-linux-gnu/:/share/apps/libc6_2.17/usr/lib64/:/share/apps/gcc-6.2.0/lib64:/share/apps/gcc-6.2.0/lib:/share/apps/python-3.6.0-shared/lib:/share/apps/cuda-8.0/lib64:/share/apps/cuda-8.0/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+export PYTHONPATH="${PYTHONPATH}:/home/guotwang/tf_project/fetal_brain_seg/NiftyNet"
+
+# iou evaluation
+#/share/apps/libc6_2.17/lib/x86_64-linux-gnu/ld-2.17.so --library-path $TF_LD_LIBRARY_PATH $(command -v /home/guotwang/miniconda3/bin/python3) /home/guotwang/tf_project/fetal_brain_seg/Demic/util/iou_evaluation.py /home/guotwang/tf_project/fetal_brain_seg/config/evaluate.txt
+
+# centroid distance evaluation
+#/share/apps/libc6_2.17/lib/x86_64-linux-gnu/ld-2.17.so --library-path $TF_LD_LIBRARY_PATH $(command -v /home/guotwang/miniconda3/bin/python3) /home/guotwang/tf_project/fetal_brain_seg/Demic/util/detect_dis_evaluation.py /home/guotwang/tf_project/fetal_brain_seg/config/evaluate.txt
+
+# Dice evaluation
+#/share/apps/libc6_2.17/lib/x86_64-linux-gnu/ld-2.17.so --library-path $TF_LD_LIBRARY_PATH $(command -v /home/guotwang/miniconda3/bin/python3) /home/guotwang/tf_project/fetal_brain_seg/Demic/util/dice_evaluation.py /home/guotwang/tf_project/fetal_brain_seg/config/evaluate.txt
+
+# hausdorff evaluation
+/share/apps/libc6_2.17/lib/x86_64-linux-gnu/ld-2.17.so --library-path $TF_LD_LIBRARY_PATH $(command -v /home/guotwang/miniconda3/bin/python3) /home/guotwang/tf_project/fetal_brain_seg/Demic/util/hausdorff_evaluation.py /home/guotwang/tf_project/fetal_brain_seg/config/evaluate.txt

bash/train.sh

@@ -0,0 +1,27 @@
+#$ -P gpu
+#$ -l gpu=1
+#$ -l gpu_pascal=1
+#$ -l h_rt=10:0:0
+#$ -l tmem=10.0G
+##$ -l h=tesla5
+#$ -N unet3d
+#$ -S /bin/bash
+#$ -wd /home/guotwang/tf_project/fetal_brain_seg/bash
+#!/bin/bash
+# The lines above are resource requests. This script has requested 1 Titan X (Pascal) GPU for 24 hours, and 11.5 GB of memory to be started with the BASH Shell.
+# More information about resource requests can be found at http://hpc.cs.ucl.ac.uk/job_submission_sge/
+
+# This line ensures that you only use the 1 GPU requested.
+nvidia-smi
+export CUDA_VISIBLE_DEVICES=$(( `nvidia-smi | grep " / .....MiB"|grep -n " ...MiB / [0-9]....MiB"|cut -d : -f 1|head -n 1` - 1 ))
+echo $CUDA_VISIBLE_DEVICES
+if [ $CUDA_VISIBLE_DEVICES -lt 0 ];then
+exit 1
+fi
+# These lines runs your NiftyNet task with the correct library paths for tensorflow
+TF_LD_LIBRARY_PATH=/share/apps/libc6_2.17/lib/x86_64-linux-gnu/:/share/apps/libc6_2.17/usr/lib64/:/share/apps/gcc-6.2.0/lib64:/share/apps/gcc-6.2.0/lib:/share/apps/python-3.6.0-shared/lib:/share/apps/cuda-8.0/lib64:/share/apps/cuda-8.0/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+export PYTHONPATH="${PYTHONPATH}:/home/guotwang/tf_project/fetal_brain_seg/NiftyNet:/home/guotwang/tf_project/fetal_brain_seg"
+
+#/share/apps/libc6_2.17/lib/x86_64-linux-gnu/ld-2.17.so --library-path $TF_LD_LIBRARY_PATH $(command -v /home/guotwang/miniconda3/bin/python3) /home/guotwang/tf_project/fetal_brain_seg/Demic/train_test/model_train.py /home/guotwang/tf_project/fetal_brain_seg/config/detect/train_unet3d.txt
+
+/share/apps/libc6_2.17/lib/x86_64-linux-gnu/ld-2.17.so --library-path $TF_LD_LIBRARY_PATH $(command -v /home/guotwang/miniconda3/bin/python3) /home/guotwang/tf_project/fetal_brain_seg/Demic/train_test/model_train.py /home/guotwang/tf_project/fetal_brain_seg/config/segment/train_unet-ml.txt

config/detect/train_pnet.txt

@@ -0,0 +1,116 @@
+[dataset]
+# filename of tfrecords
+# (required, string)
+data_train = [/home/guotwang/data/FetalBrain/tf_records/hp_train.tfrecords]
+
+data_valid0 = [/home/guotwang/data/FetalBrain/tf_records/h_valid.tfrecords]
+data_valid1 = [/home/guotwang/data/FetalBrain/tf_records/p_valid.tfrecords]
+
+[sampler]
+# batch size to load
+# (optional, integer, default is 5)
+batch_size  = 1
+
+# patch shape [D, H, W, C]
+# (required, list)
+data_shape  = [20, 96, 96, 1]
+weight_shape= [20, 96, 96, 1]
+label_shape = [20, 96, 96, 1]
+
+# whether load ground truth or not
+# (optional, bool, default is False)
+with_ground_truth = True
+
+# data augmentation
+# a list with angles in radians, e.g. [-3.14, 3.14]
+# (optional, list of float, default is None)  enabled when patch_mode = 1
+random_rotate = [-3.14, 3.14]
+
+# a bool denoting left right flip or not
+# (optional, bool, default is False)
+flip_left_right = True
+
+# a bool denoting up down flip or not
+# (optional, bool, default is False)
+flip_up_down = True
+
+# two lists of label convert, each label in label_convert_source is converted to the corresponding one in label_convert_target, used for segmentation task
+# (optional, list, default is None)
+label_convert_source = [0, 1, 2, 3, 4]
+label_convert_target = [0, 1, 0, 0, 0]
+
+# patch sampling mode
+# 0: for segmentation, randomly sample patch with fixed size,
+# 1: for segmentation, crop with bounding box and resize within plane,
+#    and randomly sample along z axis
+# 2: resize 2d images to given size
+# 3: randomly using 1 and 2
+# (required, integer)
+patch_mode = 2
+
+# bounding box margin for along z,y,x axis. (when patch_mode = 1 )
+# (optional, list of integers, default is [0,0,0])
+bounding_box_margin = [3,8,8]
+
+[network]
+# type of network
+# (required, string)
+net_type = PNet
+
+# name of network
+# (required, string)
+net_name = pnet
+
+# bn_training, True: use batch mean False: use moving mean
+# (optional, bool, default: True)
+bn_training = True
+
+# number of class, required for segmentation task
+# (optional, integer)
+class_num = 2
+
+[network_parameter]
+
+[training]
+# application type, 0 segmentation, 1 regression
+# (required, integer)
+app_type = 0
+
+# seed of random number, an integer
+# (optional, integer, default is 0)
+random_seed = 0
+
+# start epoch
+# (required, integer)
+start_iter = 0
+
+# maximal epoch
+# (required, integer)
+maximal_iter = 5000
+
+# snapshot epoch
+# (required, integer)
+snapshot_iter = 1000
+
+# pertained model, used for fine-tuning, required if start_epoch > 0
+# (optional, string, default is None)
+pretrained_model = 
+
+# model_save_prefix
+# (required, string)
+model_save_prefix = model/detect/pnet_test
+
+
+test_interval = 50
+
+# number of batch for testing in each epoch
+# (required, integer)
+test_steps = 2
+
+# learning rate
+# (optional, float, default is 1e-3)
+learning_rate = 1e-3
+
+# weight decay
+# (optional, float, default is 1e-7)
+decay = 1e-7

config/segment/train_pnet-s10-ml.txt

@@ -0,0 +1,116 @@
+[dataset]
+# filename of tfrecords
+# (required, string)
+data_train = [/mnt/shared/guotai/data/tf_records/hp_train.tfrecords]
+
+data_valid0 = [/mnt/shared/guotai/data/tf_records/h_valid.tfrecords]
+data_valid1 = [/mnt/shared/guotai/data/tf_records/p_valid.tfrecords]
+
+[sampler]
+# batch size to load
+# (optional, integer, default is 5)
+batch_size  = 1
+
+# patch shape [D, H, W, C]
+# (required, list)
+data_shape  = [10, 96, 96, 1]
+weight_shape= [10, 96, 96, 1]
+label_shape = [10, 96, 96, 1]
+
+# whether load ground truth or not
+# (optional, bool, default is False)
+with_ground_truth = True
+
+# data augmentation
+# a list with angles in radians, e.g. [-3.14, 3.14]
+# (optional, list of float, default is None)
+random_rotate = [-3.14, 3.14]
+
+# a bool denoting left right flip or not
+# (optional, bool, default is False)
+flip_left_right = True
+
+# a bool denoting up down flip or not
+# (optional, bool, default is False)
+flip_up_down = True
+
+# two lists of label convert, each label in label_convert_source is converted to the corresponding one in label_convert_target, used for segmentation task
+# (optional, list, default is None)
+label_convert_source = [0, 1, 2, 3, 4]
+label_convert_target = [0, 1, 0, 0, 0]
+
+# patch sampling mode
+# 0: for segmentation, randomly sample patch with fixed size,
+# 1: for segmentation, crop with bounding box and resize within plane,
+#    and randomly sample along z axis
+# 2: for regression, resize image to fixed size,
+#    resize 2d images to given size, and get spatial transformer parameters
+# (required, integer)
+patch_mode = 1
+
+# bounding box margin for along z,y,x axis. (when patch_mode = 1 or 2)
+# (optional, list of integers, default is [0,0,0])
+bounding_box_margin = [5,20,20]
+
+[network]
+# type of network
+# (required, string)
+net_type = PNet
+
+# name of network
+# (required, string)
+net_name = pnet
+
+# bn_training, True: use batch mean False: use moving mean
+# (optional, bool, default: True)
+bn_training = True
+
+# number of class, required for segmentation task
+# (optional, integer)
+class_num = 2
+
+[network_parameter]
+
+[training]
+# application type, 0 segmentation, 1 regression
+# (required, integer)
+app_type = 0
+
+# seed of random number, an integer
+# (optional, integer, default is 0)
+random_seed = 0
+
+# start epoch
+# (required, integer)
+start_iter = 0
+
+# maximal epoch
+# (required, integer)
+maximal_iter = 5000
+
+# snapshot epoch
+# (required, integer)
+snapshot_iter = 500
+
+# pertained model, used for fine-tuning, required if start_epoch > 0
+# (optional, string, default is None)
+pretrained_model = 
+
+multi_scale_loss = True
+# model_save_prefix
+# (required, string)
+model_save_prefix = model/pnet-s10-ml/pnet
+
+test_interval = 50
+
+# number of batch for testing in each epoch
+# (required, integer)
+test_steps = 2
+
+# learning rate
+# (optional, float, default is 1e-3)
+learning_rate = 1e-3
+
+# weight decay
+# (optional, float, default is 1e-7)
+decay = 1e-7