Commit a75f79d2 authored by mathpluscode's avatar mathpluscode

train pipeline works


preprocess pipeline works
parent 65ab7beb
# Pycharm
/.idea
# IDE
.idea
*.code-workspace
.vscode/
# Generated files
.tox
......@@ -10,3 +12,8 @@
# PyInstaller
build/
dist/
# ignore data log folder
!data/
data/*
log/
\ No newline at end of file
dir:
data: "data/"
log: "log/"
data:
cv: # cross validation
train: -1 # -1 means do full cv, >= 0 means just take the specific run
leave_out: 1 # num of folders for test set
overfit: -1 # >= 0 means train and evaluate on the same set of folders
preprocess:
shape: # shape is [height, width]
orig: [540, 1920]
border: 130 # crop border manually, output has shape [540, 1660]
input: [64, 64] # input size for NN
sl:
keep_ratio: 1 # proportion of labeled data, randomly sampled
ssl:
activate: true
fps: 4 # frequency of frame extraction
skip: 1 # is skip = 2, means take 1, 3, 5-th frames of all unlabeled data, -1 means do not use unlabeled data
aug:
contrast:
prob: 0.75
lower: 0.8
upper: 1.2
brightness:
prob: 0.75
max_delta: 0.1
standardize: true # (image - mean) / std, mean and std is one value per color channel, doesn't depend on pixel location
affine:
prob: 0.75 # perform affine transformation with such probability
scale: 0.1 # scale for affine transformation
model:
network:
name: "unet"
unet:
norm: 'layer_norm' # batch_norm or layer_norm or none
depth: 4 # depth of NN
size: 16 # number of filters in conv depends on this parameter
deep_input: true # input at deep layers, except bottom
attn: false
loss:
name: "dice" # cross_entropy or dice
dice:
pos_pixel_weight: 1.0 # weight of positive pixel compared to negative pixel
sample_weight: 1.0 # weight of a sample in a batch is exp(weight * loss), get normalized later, 0 means no adjustment
opt:
optimizer: "adam" # adam or adamw
lr: 1.e-4 # learning rate
clip_norm: 5.0 # grad clip norm
weight_decay: 1.e-5 # l2 loss weight for adam or weight_decay for adamw
batch_size: 4
ssl:
mode: "mean_teacher" # pi: consistency between noised1 and noised2, mean_teacher: noise1 for student and noise2 for teacher
loss:
name: "dice" # dice or mse
weighted: false # if true, the sample weight in the loss comes from the teacher
loss_max_weight: 0.1 # max weight of unsupervised loss
ramp_step: 1000 # exponential ramp
mean_teacher:
decay:
ramp: 0.99 # ema decay during ramp up phase
train: 0.999 # ema decay after ramp up phase
tf: # configuration for tensorflow
gpu: "" # GPU ID should be specified manually
num_parallel_calls: 4 # number of cpus
save_best:
metric_name: "metric_bin/f1_percentile_50"
large_is_better: true
run: # tf.estimator.RunConfig
save_checkpoints_steps: 100 # save checkpoints every 1000 steps
keep_checkpoint_max: 5 # retain the x most recent checkpoints
log_step_count_steps: 100 # output frequency
train: # tf.estimator.TrainSpec
max_steps: 100
eval: # tf.estimator.EvalSpec
steps: 200 # number of batches for evaluation, each folder has at maximum 260 frames
start_delay_secs: 0 # Start evaluating after waiting for this many seconds
throttle_secs: 120 # Do not re-evaluate unless the last evaluation was started at least this many seconds ago.
#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys
from yfmil3id2019.ui.preprocess_command_line import gen_mean_std
if __name__ == "__main__":
sys.exit(gen_mean_std(sys.argv[1:]))
#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys
from yfmil3id2019.ui.preprocess_command_line import gen_unlabel
if __name__ == "__main__":
sys.exit(gen_unlabel(sys.argv[1:]))
......@@ -3,3 +3,11 @@
# confused with the software requirements, which are listed in
# doc/requirements.rst
numpy
scipy
matplotlib
pyyaml
opencv-python
tqdm
seaborn
ray
plotly
......@@ -2,29 +2,7 @@
""" yfmil3id2019 tests"""
from yfmil3id2019.ui.yfmil3id2019_train_app import run_app
from yfmil3id2019.algorithms import addition, multiplication
from yfmil3id2019.ui.train_app import train_app
import six
# Pytest style
def test_using_pytest_yfmil3id2019():
x = 1
y = 2
verbose = False
multiply = False
expected_answer = 3
assert run_app(x, y, multiply, verbose) == expected_answer
def test_addition():
assert addition.add_two_numbers(1, 2) == 3
def test_multiplication():
assert multiplication.multiply_two_numbers(2, 2) == 4
......@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
import sys
from yfmil3id2019.ui.yfmil3id2019_train_command_line import main
from yfmil3id2019.ui.train_command_line import main
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
# coding=utf-8
""" Module for adding numbers. """
def add_two_numbers(input_x, input_y):
""" Add two numbers """
return input_x + input_y
# coding=utf-8
""" Module for multiplying numbers. """
def multiply_two_numbers(input_x, input_y):
""" Multiply two numbers """
return input_x * input_y
import argparse
import logging
import os
import matplotlib
import numpy as np
import tensorflow as tf
import yaml
from yfmil3id2019.src.model.metric import seg_metric_np
from yfmil3id2019.src.util import make_dir
class CrossValidationRun:
def __init__(self, folders_lbl_train, folders_unlbl_train, folders_lbl_eval):
self.folders_lbl_train = folders_lbl_train
self.folders_unlbl_train = folders_unlbl_train
self.folders_lbl_eval = folders_lbl_eval
self.mean_std_folders = [x.replace('labeled', 'mean_std') for x in folders_lbl_train]
def __repr__(self):
return get_folder_name_from_paths(self.folders_lbl_train) + '/' \
+ (get_folder_name_from_paths(self.folders_unlbl_train) if self.folders_unlbl_train is not None else '') + '/' \
+ get_folder_name_from_paths(self.folders_lbl_eval)
def generate_mean_std(self, dir_run):
matplotlib.use('agg')
import matplotlib.pyplot as plt
from matplotlib.image import imsave
# init lists
means = []
stds = []
weights = []
# read files
for folder in self.mean_std_folders:
mean = plt.imread(folder + '/train_mean.png')
std = plt.imread(folder + '/train_std.png')
mean = mean[:, :, :3]
std = std[:, :, :3]
with open(folder + '/num_img.txt', 'r') as f:
num = int(f.read().splitlines()[0])
means.append(mean)
stds.append(std)
weights.append(num)
# calculate
mean *= 0
std *= 0
total_weight = np.sum(weights)
weights = [x / total_weight for x in weights]
for i in range(len(weights)):
mean += means[i] * weights[i]
std += (stds[i] ** 2) * weights[i]
std = np.sqrt(std)
# save file
mean_std_path = [dir_run + '/mean.png', dir_run + '/std.png']
imsave(mean_std_path[0], mean, vmin=0, vmax=1)
imsave(mean_std_path[1], std, vmin=0, vmax=1)
return mean_std_path
def split_train_eval(folders_labeled, folders_unlabeled, param):
# retrieve parameters
train = param['train']
leave_out = param['leave_out']
overfit = param['overfit']
num_folders = len(folders_labeled)
# extend labeled folders
# for instance we have folder [0, 1, 2, 3, 4, 5, 6]
# if the leave_out is 2
# the train folders will be [0, 1], [2, 3], [4, 5, 6]
runs = []
indices = [x for x in range(num_folders)]
indicies_evals = np.array_split(np.arange(num_folders), num_folders // leave_out)
for run_id in range(num_folders // leave_out):
indicies_eval = list(indicies_evals[run_id])
indicies_train = [x for x in indices if x not in indicies_eval]
fs_lbl_train = [folders_labeled[x] for x in indicies_train]
fs_unlbl_train = [folders_unlabeled[x] for x in indicies_train] if folders_unlabeled is not None else None
fs_lbl_eval = [folders_labeled[x] for x in indicies_eval]
runs.append(CrossValidationRun(fs_lbl_train, fs_unlbl_train, fs_lbl_eval))
if train >= 0:
runs = [runs[train]]
if overfit >= 0:
run = runs[overfit]
run.folders_lbl_eval = run.folders_lbl_train
runs = [run]
return runs
def get_folder_name_from_paths(paths):
return '_'.join([x.split('/')[-1] for x in paths])
def save_predict_results(results, dir_run, name):
matplotlib.use('agg')
import matplotlib.pyplot as plt
from matplotlib.image import imsave
dir_pred = dir_run + '/preds/%s/' % name
make_dir(dir_pred)
with open(dir_pred + 'metric.log', 'w+') as f:
for i, result in enumerate(results):
images = result['images']
masks = result['masks'] # [0,1]
preds = result['preds'] # [0,1]
logits = result['logits'] # [0,1]
images = (images - np.min(images)) / (np.max(images) - np.min(images))
imsave(dir_pred + '/%d_image.png' % i, images)
imsave(dir_pred + '/%d_mask.png' % i, masks, vmin=0, vmax=1, cmap='gray')
imsave(dir_pred + '/%d_prob.png' % i, preds, vmin=0, vmax=1, cmap='gray')
imsave(dir_pred + '/%d_pred.png' % i, np.round(preds), vmin=0, vmax=1, cmap='gray')
metrics = seg_metric_np(preds, masks)
line = '%d|' % i
for k, v in metrics.items():
line += k + '=' + '%f,' % v
f.write(line + '\n')
"""functions used for data augmentation"""
import random
import numpy as np
import tensorflow as tf
from yfmil3id2019.src.util import assert_tf_rank
from yfmil3id2019.src.wrapper.layer import flatten
def apply_augmentation_in_model(x, mean, std, ch, affine, config_aug, mode):
"""
assuming in the training mode
:param x: shape = [batch_size, H, W, C]
:param mean: shape = [1,1,1,3]
:param std: shape = [1,1,1,3]
:param affine:
:param config_aug: config['aug']
:param mode:
"""
assert_tf_rank(x, 4)
if mode == tf.estimator.ModeKeys.TRAIN:
# adjust color
x = adjust_color(x, ch, config_aug)
# standardize
if config_aug['standardize']:
x = standardize(x, mean, std, ch)
if mode == tf.estimator.ModeKeys.TRAIN:
# affine, should also be the last augmentation
p = config_aug['affine']['prob']
scale = config_aug['affine']['scale']
if affine:
if scale > 0:
if np.random.rand() < p:
x = apply_affine_transform(x, scale, return_fn=False)
x = tf.stop_gradient(x)
return x
def adjust_color(x, ch, config_aug):
"""
:param x:
:param ch:
:param config_aug:
"""
def _adjust_color(x):
"""perform at most one color adjustment for speed"""
color_adjusted = True
if np.random.rand() < config_aug['contrast']['prob']:
x = tf.image.random_contrast(image=x, lower=config_aug['contrast']['lower'], upper=config_aug['contrast']['upper'])
elif np.random.rand() < config_aug['brightness']['prob']:
x = tf.image.random_brightness(image=x, max_delta=config_aug['brightness']['max_delta'])
else:
color_adjusted = False
if color_adjusted:
x = tf.clip_by_value(x, 0.0, 1.0)
return x
assert_tf_rank(x, 4)
if ch == 3:
# first three channels are test image
x = _adjust_color(x)
elif ch == 4:
# first three channels are labeled image, last channel is label
image, mask = x[:, :, :, :-1], x[:, :, :, -1:]
image = _adjust_color(image)
x = tf.concat([image, mask], axis=-1)
elif ch == 7:
# first three channels are unlabeled image, next three channels are labeled image, last channel is label
image1, image2, mask = x[:, :, :, :3], x[:, :, :, 3:6], x[:, :, :, -1:]
image1 = _adjust_color(image1)
image2 = _adjust_color(image2)
x = tf.concat([image1, image2, mask], axis=-1)
else:
raise ValueError('Unknown input channel %d.' % ch)
return x
def standardize(x, mean, std, ch):
assert_tf_rank(x, 4)
if ch == 3:
# first three channels are test image
x = (x - mean) / std
elif ch == 4:
# first three channels are labeled image, last channel is label
image, mask = x[:, :, :, :-1], x[:, :, :, -1:]
image = (image - mean) / std
x = tf.concat([image, mask], axis=-1)
elif ch == 7:
# first three channels are unlabeled image, next three channels are labeled image, last channel is label
image1, image2, mask = x[:, :, :, :3], x[:, :, :, 3:6], x[:, :, :, -1:]
image1 = (image1 - mean) / std
image2 = (image2 - mean) / std
x = tf.concat([image1, image2, mask], axis=-1)
else:
raise ValueError('Unknown input channel %d.' % ch)
return x
def apply_affine_transform(images, scale, return_fn):
"""
:param images: shape = [None, H, W, C]
:param scale:
:param return_fn: if True, return a function to apply the same transform
:return:
"""
assert_tf_rank(images, 4)
sh = images.get_shape().as_list()
batch_size = sh[0]
size = sh[1:3]
A = get_affine_transform_batch(size, scale, batch_size) # shape = [batch_size, 2, 3]
A = flatten(A) # shape = [batch_size, 6]
A = tf.concat([A, tf.zeros([batch_size, 2], tf.float32)], axis=1) # shape = [batch_size, 8]
images = tf.contrib.image.transform(images=images, transforms=tf.convert_to_tensor(A))
return images
def calculate_affine_matrix(orig, new):
"""
:param orig: shape = [4, 3]
:param new: shape = [4, 2]
:return: shape = [2, 3]
"""
A = np.linalg.lstsq(orig, new, rcond=-1)[0].T # shape=[2, 3]
return A
def get_joint_affine_transform_batch(size, scale, batch_size):
"""
get affine transforms A1, A2, A12 such that
apply A1, then A12 is equivalent to apply A2
they all have shape = [batch_size, 8]
"""
A = get_affine_transform_batch(size=size, scale=scale, batch_size=batch_size * 2) # [batch_size*2, 2, 3], np array
A = np.concatenate([A, np.zeros([batch_size * 2, 1, 3])], axis=1) # [batch_size*2, 3, 3], np array
A[:, 2, 2] = 1
A = tf.convert_to_tensor(A, dtype=tf.float32)
A1 = A[:batch_size, :, :] # shape = scale
A12 = A[batch_size:, :, :] # shape = [batch_size, 3, 3]
A2 = tf.matmul(A12, A1) # A2(x) = A12(A1(x))
A1 = flatten(A1)[:, :8]
A12 = flatten(A12)[:, :8]
A2 = flatten(A2)[:, :8]
return A1, A2, A12
def get_affine_transform_batch(size, scale, batch_size):
"""
generate a random affine transformation
affine transformation
[[x'] = [[* * *] [[x]
[y']] [* * *] [y]
[1]] [0 0 1]] [1]]
to calculate the transformaiton parameters
- each corner of the image is shifted randomly within a given range
- a linear system is solved to calculate the approximate affine transformation that
maps the original corners to the shifted ones
- here, the transformation is only affine, not projective
:param size: [H, W]
:param scale: (0, 1)
:param batch_size:
:return: shape = [batch_size, 2, 3]
"""
W = size[1] / size[0]
H = 1
coords_orig = np.array([[[W / 2, H / 2],
[-W / 2, H / 2],
[-W / 2, -H / 2],
[W / 2, -H / 2]]], dtype=np.float32) # [1, 4, 2]
coords_orig = np.tile(coords_orig, (batch_size, 1, 1)) # [batch_size, 4, 2]
offset = np.random.uniform(-scale, scale, [batch_size, 4, 2]) # [batch_size, 4, 2]
coords_new = coords_orig + offset # [batch_size, 4, 2]
coords_orig = np.concatenate([coords_orig, np.ones((batch_size, 4, 1))], axis=2) # [batch_size, 4, 3]
A = np.stack([calculate_affine_matrix(coords_orig[k, :, :], coords_new[k, :, :]) for k in range(batch_size)], axis=0) # [batch_size, 2, 3]
A = A.astype(np.float32)
return A
"""
Functions used for preparing data.
"""
import os
import random
import numpy as np
import tensorflow as tf
from yfmil3id2019.src.wrapper.layer import resize_image
def get_labeled_folders(cwd, config, training):
"""get the paths of folders containing labeled data
:param cwd: current working directory
:param config: dict of the config file
:param training: false if it is preprocessing
:return: paths of folders
"""
data_dir = cwd + config["dir"]["data"] + 'img/labeled'
folders = [f.path for f in os.scandir(data_dir) if f.is_dir()]
folders = [x for x in folders if 'HLS' not in x] # remove folders containing HLS in the name
folders = sorted(folders)
return folders
def get_unlabeled_folders(folders_labeled, config):
"""
given folder paths of labeled data
return the corresponding folders for unlabeled data
the path is specified using fps in config
:param folders_labeled: list of paths, each element is like '[...]/LiverSeg/data/img/labeled/LR01'
:param config:
:return: same order as folders_labeled, but some folders might not exist
"""
if not config['data']['ssl']['activate']:
return None
folders = [x.replace('labeled', 'unlabeled/fps%d' % config['data']['ssl']['fps']) for x in folders_labeled]
return folders
def decode_png(path, channels):
"""read path and return the tensor
:param path: path of file
:param channels: 3 for image, 1 for mask
:return: tf tensor
"""
x = tf.image.decode_png(tf.io.read_file(path), channels=channels)
x = tf.cast(x, tf.float32) / 255.0
return x
def extract_image_mask_fnames(folders, has_mask, keep_ratio, skip):
"""
:param folders: a list of folder paths
:param has_mask: a boolean indicating if we are looking for mask as well
:param keep_ratio: sample iff > 0
:param skip: skip frames iff > 1
"""
if folders is None:
return None
img_fnames = []
for folder_path in folders:
subfolders = [f.path + "/" for f in os.scandir(folder_path) if f.is_dir()] # extract paths of subfolders, some docx, zip files are ignored
for subfolder in subfolders:
img_fnames_sub = []
fnames = sorted(os.listdir(subfolder))
xmls = [x[:-11] for x in fnames if x.endswith("Contour.xml")]
imgs = [x[:-4] for x in fnames if x.endswith(".png") and not x.endswith("Mask.png")]
if has_mask and (len(xmls) == 0):
# in some folders, there's no xml file and mask imgs are all zero
continue
for x in imgs:
if has_mask: # label is correct iff the mask and contour file both exist
if x not in xmls:
continue
if x + "Mask.png" not in fnames:
continue
img_fnames_sub.append(subfolder + x)
if keep_ratio > 0:
random.seed(0)
random.shuffle(img_fnames_sub)
random.seed(None)
keep_len = int(keep_ratio * len(img_fnames_sub))
keep_len = max(keep_len, 2)
img_fnames_sub = img_fnames_sub[:keep_len]
if skip > 1:
skip = int(skip)
img_fnames_sub = img_fnames_sub[::skip]
img_fnames += img_fnames_sub
return img_fnames
def load_image_mask(img_path, has_mask, preprocess):
"""
load image from path and perform basic preprocessing i.e. cut border
:param img_path:
:param preprocess: dictionary of params
:return:
"""
# read file
image = decode_png(img_path + ".png", channels=3)
if has_mask:
mask = decode_png(img_path + "Mask.png", channels=1)
x = tf.concat([image, mask], axis=-1)
else:
x = image
# cut black border
width, border_width = preprocess['shape']['orig'][1], preprocess['shape']['border']
x = x[:, border_width:(width - border_width), :]
x = resize_image(images=x, size=preprocess['shape']['input'])
return x
def load_dataset(image_fnames, has_mask, preprocess, num_parallel_calls):
if image_fnames is None:
return None
ds = tf.data.Dataset.from_tensor_slices(image_fnames)
ds = ds.map(lambda x: load_image_mask(x, has_mask, preprocess), num_parallel_calls=num_parallel_calls)
return ds
def build_dataset(folders_lbl, folders_unlbl, mean_std_path, training, config):
"""
:param folders_lbl:
:param folders_unlbl: None if SSL is not activated
:param training:
:param config:
:return:
"""
def shuffle_and_repeat(dataset, buffer_size):
dataset = dataset.shuffle(buffer_size=buffer_size)
dataset = dataset.repeat()
return dataset
# extract params
preprocess = config['data']['preprocess']
batch_size = config['model']['opt']['batch_size']
num_parallel_calls = config['tf']['num_parallel_calls']
keep_ratio = config['data']['sl']['keep_ratio'] if training else -1
ssl_activate = config['data']['ssl']['activate']
skip = config['data']['ssl']['skip']
# extract paths of images
imgs_lbl = extract_image_mask_fnames(folders=folders_lbl, has_mask=True, keep_ratio=keep_ratio, skip=-1)
if ssl_activate:
if skip >= 1: # use ssl with unlabeled data
tf.logging.info("SSL is activate with unlabeled data")
imgs_unlbl = extract_image_mask_fnames(folders=folders_unlbl, has_mask=False, keep_ratio=-1, skip=skip)
else: # use ssl but without unlabeled data
tf.logging.info("SSL is activate but without unlabeled data")
imgs_unlbl = imgs_lbl
else: # ssl not activate
tf.logging.info("SSL is not activate")
imgs_unlbl = None
# build dataset
ds_lbl = load_dataset(imgs_lbl, has_mask=True, preprocess=preprocess, num_parallel_calls=num_parallel_calls)
ds_unlbl = load_dataset(imgs_unlbl, has_mask=False, preprocess=preprocess, num_parallel_calls=num_parallel_calls)
# read mean std images
# read it now so that no need to read it for every image
mean = decode_png(mean_std_path[0], channels=3)
std = decode_png(mean_std_path[1], channels=3)
mean = tf.expand_dims(mean[:1, :1, :], axis=0) # shape = [1,1,1,3]
std = tf.expand_dims(std[:1, :1, :], axis=0) # shape = [1,1,1,3]
if training:
# shuffle repeat DA
ds_lbl = shuffle_and_repeat(ds_lbl, buffer_size=batch_size * 2)
if ds_unlbl is not None:
ds_unlbl = shuffle_and_repeat(ds_unlbl, buffer_size=batch_size * 16)
dataset = tf.data.Dataset.zip((ds_unlbl, ds_lbl))
dataset = dataset.map(lambda x, y: tf.concat([x, y], axis=-1), num_parallel_calls=num_parallel_calls)
else: