Commit db0ddbeb authored by mathpluscode's avatar mathpluscode

clean code to satisfy pep8

parent 8aa2bed4
Pipeline #3307 failed with stages
in 4 minutes and 40 seconds
......@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
import sys
from yfmil3id2019.ui.eval_command_line import eval
from yfmil3id2019.ui.eval_command_line import evaluate
if __name__ == "__main__":
sys.exit(eval(sys.argv[1:]))
sys.exit(evaluate(sys.argv[1:]))
......@@ -22,5 +22,5 @@ opencv-python
tqdm
seaborn
ray
plotly
plotly==4.1.0
tensorflow-gpu
......@@ -10,5 +10,5 @@ opencv-python
tqdm
seaborn
ray
plotly
plotly==4.1.0
tensorflow-gpu
\ No newline at end of file
......@@ -32,7 +32,7 @@ unsafe-load-any-extension=no
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code
extension-pkg-whitelist=numpy
extension-pkg-whitelist=numpy,cv2
# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
......@@ -99,7 +99,7 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme
[BASIC]
# Good variable names which should always be accepted, separated by a comma
good-names=i,j,k,ex,Run,_
good-names=i,j,k,v,f,ex,Run,_
# Bad variable names which should always be refused, separated by a comma
bad-names=foo,bar,baz,toto,tutu,tata
......
This diff is collapsed.
# coding=utf-8
"""
Parse parameters to analyze the evaluation results
"""
import argparse
import os
import yaml
from yfmil3id2019.ui.analyse_app import analyse_stats, analyse_foreground, analyse_foreground_all
from yfmil3id2019.ui.analyse_app import analyse_stats, analyse_foreground, \
analyse_foreground_all
def analyse(args=None):
"""
:param args:
:return:
"""
parser = argparse.ArgumentParser(description='yfmil3id2019_analyse')
parser.add_argument('-p',
......@@ -17,7 +22,8 @@ def analyse(args=None):
parser.add_argument('--best',
dest='best',
action='store_true',
help='use the best model instead of the final one, otherwise use the last checkpoint')
help='use the best model instead of the final one, '
'otherwise use the last checkpoint')
parser.add_argument('--all',
dest='all',
action='store_true',
......@@ -33,7 +39,7 @@ def analyse(args=None):
args = parser.parse_args(args)
analyse_stats(log_path=args.path, best=args.best, all=args.all)
analyse_stats(log_path=args.path, best=args.best, eval_train=args.all)
if not args.all:
analyse_foreground(log_path=args.path, best=args.best, std=args.std)
else:
......
This diff is collapsed.
"""
Parse parameters to evaluate models
"""
import argparse
import logging
import os
......@@ -6,7 +10,12 @@ import yaml
from yfmil3id2019.ui.eval_app import eval_app
def eval(args=None):
def evaluate(args=None):
"""
:param args:
:return:
"""
logging.getLogger("tensorflow").setLevel(logging.FATAL)
# parse args
......@@ -26,7 +35,8 @@ def eval(args=None):
parser.add_argument('--best',
dest='best',
action='store_true',
help='use the best model instead of the final one, otherwise use the last checkpoint')
help='use the best model instead of the final one, '
'otherwise use the last checkpoint')
parser.add_argument('--eval',
dest='eval',
action='store_true',
......@@ -52,12 +62,13 @@ def eval(args=None):
path = path[:-1]
folders_cv = [f.path for f in os.scandir(path) if f.is_dir()]
config_path = folders_cv[0] + '/config_backup.yaml'
with open(config_path) as file:
config = yaml.load(file)
with open(config_path) as f:
config = yaml.load(f)
# modify config
config['model']['opt']['batch_size'] = args.bs
config['tf']['gpu'] = args.gpu
os.environ['CUDA_VISIBLE_DEVICES'] = config['tf']['gpu']
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
eval_app(config=config, path=path, best=args.best, all=args.all, eval=args.eval, return_hd=args.return_hd)
eval_app(config=config, path=path, best=args.best, eval_train=args.all,
predict=args.eval, return_hd=args.return_hd)
"""This function calculate and generates the mean and std for each patient"""
"""
Functions defined here are used to
- generate the mean and std for each patient
- extract unlabeled iamges
- perform pre-processing
"""
import os
import yaml
import numpy as np
from tqdm import tqdm
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
from matplotlib.image import imsave, imread
from yfmil3id2019.src.cv import get_folder_name_from_paths, split_train_eval
from yfmil3id2019.src.data.load import get_labeled_folders, extract_image_mask_fnames
from yfmil3id2019.src.data.load import get_labeled_folders, \
extract_image_mask_fnames
from yfmil3id2019.src.util import make_dir
def calc_mean_std(img_fnames, chunk_size=200):
def calc_mean_std(img_fnames, max_chunk_size=200):
"""
calculate mean and std for the given images
if there are too many images, load images separately
"""
# cut into chunks
chunks = [img_fnames[x:x + chunk_size] for x in range(0, len(img_fnames), chunk_size)]
chunks = [img_fnames[x:x + max_chunk_size] for x in
range(0, len(img_fnames), max_chunk_size)]
# process chunks
means, stds, nums = [], [], []
for chunk in chunks:
# load
stacked = np.stack([plt.imread(x + '.png').astype(np.float32) for x in chunk], axis=3) # imread from plt returns value between [0, 1], shape=(H, W, 3, None)
stacked = np.stack(
[imread(x + '.png').astype(np.float32) for x in chunk],
axis=3) # values in [0, 1], shape=(H, W, 3, None)
# cut border
sh = stacked.shape
W = sh[1]
B = 130
stacked = stacked[:, B:(W - B), :, :]
img_width = stacked.shape[1]
border_width = 130
stacked = stacked[:, border_width:(img_width - border_width), :, :]
# calculate
mean = np.mean(stacked, axis=(0, 1, 3)).reshape((1, 1, 3))
std = np.std(stacked, axis=(0, 1, 3)).reshape((1, 1, 3))
N = len(chunk)
chunk_size = len(chunk)
# log stats
means.append(mean)
stds.append(std)
nums.append(N)
nums.append(chunk_size)
# merge
mean = means[0] * 0
std = stds[0] * 0
for i in range(len(means)):
for i, _ in enumerate(means):
mean += means[i] * nums[i]
std += (stds[i] ** 2) * nums[i]
mean = mean / np.sum(nums)
......@@ -54,9 +56,11 @@ def calc_mean_std(img_fnames, chunk_size=200):
def gen_mean_std_app(config):
# init
with open("config.yaml") as file:
config = yaml.load(file)
"""
calculate the mean and std for each patient
:param config:
:return:
"""
cwd = os.getcwd() + '/'
# file names
......@@ -65,46 +69,62 @@ def gen_mean_std_app(config):
folder_names = [x.split('/')[-1] for x in folders]
# calculate the mean of each folder
for i in range(len(folders)):
for i, _ in enumerate(folders):
dir_i = dir_mean_std + folder_names[i]
make_dir(dir_i, remove=True)
print('Preprocessing the %d-th folder %s' % (i, folder_names[i]))
img_fnames = extract_image_mask_fnames(folders=[folders[i]], has_mask=True, keep_ratio=1, skip=1)
N = len(img_fnames) # number of imgs
print('Pre-processing the %d-th folder %s' % (i, folder_names[i]))
img_fnames = extract_image_mask_fnames(folders=[folders[i]],
has_mask=True, keep_ratio=1,
skip=1)
mean, std = calc_mean_std(img_fnames)
print('Calculating train mean & std for the %d-th folder %s' % (i, folder_names[i]))
img_fnames = extract_image_mask_fnames([folders[i]], has_mask=True, keep_ratio=1, skip=1)
sh = plt.imread(img_fnames[0] + '.png').shape # shape of mean / std should be the same as the ones in the folder
assert sh[-1] == 3
ones = np.ones(sh)
print('Calculating train mean & std for the %d-th folder %s'
% (i, folder_names[i]))
img_fnames = extract_image_mask_fnames([folders[i]], has_mask=True,
keep_ratio=1, skip=1)
ones = np.ones(imread(img_fnames[0] + '.png').shape)
imsave(dir_i + '/train_mean.png', ones * mean, vmin=0, vmax=1)
imsave(dir_i + '/train_std.png', ones * std, vmin=0, vmax=1)
with open(dir_i + '/num_img.txt', 'w') as f:
f.write('%d\n' % N)
f.write('%d\n' % len(img_fnames))
def gen_unlabel_app(dir_data, fps):
REMOVE = True # remove the existing frames
"""
extract unlabeled images from each video
:param dir_data:
:param fps:
:return:
"""
dir_video = dir_data + 'video/'
dir_frame = dir_data + 'img/unlabeled/fps%s/' % fps
make_dir(dir_frame, REMOVE)
make_dir(dir_frame, remove=True) # remove the existing frames
patient_folders = os.listdir(dir_video)
for p_folder in patient_folders:
v_folders = os.listdir(dir_video + p_folder)
for v_folder in v_folders:
video_fname = [f.path for f in os.scandir(dir_video + p_folder + '/' + v_folder) if f.path.endswith('.264')][0]
video_fname = \
[f.path for f in
os.scandir(dir_video + p_folder + '/' + v_folder) if
f.path.endswith('.264')][0]
frame_path = dir_frame + p_folder + '/' + v_folder
make_dir(frame_path)
cmd = "ffmpeg -i %s -vf fps=%s %s/%%08d.png" % (video_fname, fps, frame_path)
print('Extracting frames from %s into %s' % (video_fname, frame_path))
cmd = "ffmpeg -i %s -vf fps=%s %s/%%08d.png" % (
video_fname, fps, frame_path)
print(
'Extracting frames from %s into %s' % (video_fname, frame_path))
os.system(cmd)
def cut_image_mask_app(config):
"""
cur the black border and save the results in img/cut
:param config:
:return:
"""
width = config['data']['preprocess']['shape']['orig'][1]
border_width = config['data']['preprocess']['shape']['border']
......@@ -130,4 +150,5 @@ def cut_image_mask_app(config):
image_orig = image_orig[:, border_width:(width - border_width), :]
mask_orig = mask_orig[:, border_width:(width - border_width)]
imsave(dir_cut_run + '/%d_image_cut.png' % sample_id, image_orig)
imsave(dir_cut_run + '/%d_mask_cut.png' % sample_id, mask_orig, vmin=0, vmax=1, cmap='gray')
imsave(dir_cut_run + '/%d_mask_cut.png' % sample_id, mask_orig,
vmin=0, vmax=1, cmap='gray')
"""
Parse parameters to
- generate mean and std of images
- generate unlabelled images
- cut off black borders
"""
import argparse
import yaml
from yfmil3id2019.ui.preprocess_app import gen_mean_std_app, gen_unlabel_app, cut_image_mask_app
from yfmil3id2019.ui.preprocess_app import gen_mean_std_app, gen_unlabel_app, \
cut_image_mask_app
def gen_mean_std(args=None):
"""
:param args:
:return:
"""
parser = argparse.ArgumentParser(description='yfmil3id2019_mean_std')
parser.add_argument('-p',
......@@ -15,13 +27,18 @@ def gen_mean_std(args=None):
args = parser.parse_args(args)
# load config
with open(args.path) as file:
config = yaml.load(file)
with open(args.path) as f:
config = yaml.load(f)
gen_mean_std_app(config=config)
def gen_unlabel(args=None):
"""
:param args:
:return:
"""
parser = argparse.ArgumentParser(description='yfmil3id2019_unlabel')
parser.add_argument('-p',
......@@ -36,14 +53,19 @@ def gen_unlabel(args=None):
args = parser.parse_args(args)
# load config
with open(args.path) as file:
config = yaml.load(file)
with open(args.path) as f:
config = yaml.load(f)
gen_unlabel_app(dir_data=config['dir']['data'],
fps=args.fps)
def cut_image_mask(args=None):
"""
:param args:
:return:
"""
parser = argparse.ArgumentParser(description='yfmil3id2019_unlabel')
parser.add_argument('-p',
......@@ -54,7 +76,7 @@ def cut_image_mask(args=None):
args = parser.parse_args(args)
# load config
with open(args.path) as file:
config = yaml.load(file)
with open(args.path) as f:
config = yaml.load(f)
cut_image_mask_app(config=config)
"""
Functions defined here are used to
- test a trained model, i.e. predict on a given image
"""
import os
import numpy as np
import tensorflow as tf
import cv2
import matplotlib
matplotlib.use('agg')
from matplotlib.image import imsave, imread
from yfmil3id2019.src.model.model import model_fn
from yfmil3id2019.src.util import init_log_dir, make_dir, set_tf_logger
from yfmil3id2019.src.util import make_dir
from yfmil3id2019.src.wrapper.util import ConfigProto
from yfmil3id2019.src.data.load import build_test_dataset
def save_predict_results(results, imgs, data_path):
"""
:param results:
:param imgs:
:param data_path:
:return:
"""
dir_pred = data_path + '/preds/'
make_dir(dir_pred)
......@@ -35,10 +43,14 @@ def save_predict_results(results, imgs, data_path):
image = (image - np.min(image)) / (np.max(image) - np.min(image))
imsave(dir_pred + '/%s_image.png' % img_name, image)
if mask is not None:
imsave(dir_pred + '/%s_mask.png' % img_name, mask, vmin=0, vmax=1, cmap='gray')
imsave(dir_pred + '/%s_prob.png' % img_name, pred, vmin=0, vmax=1, cmap='gray')
imsave(dir_pred + '/%s_pred.png' % img_name, np.round(pred), vmin=0, vmax=1, cmap='gray')
imsave(dir_pred + '/%s_pred_orig.png' % img_name, np.round(pred_orig), vmin=0, vmax=1, cmap='gray')
imsave(dir_pred + '/%s_mask.png' % img_name, mask, vmin=0, vmax=1,
cmap='gray')
imsave(dir_pred + '/%s_prob.png' % img_name, pred, vmin=0, vmax=1,
cmap='gray')
imsave(dir_pred + '/%s_pred.png' % img_name, np.round(pred), vmin=0,
vmax=1, cmap='gray')
imsave(dir_pred + '/%s_pred_orig.png' % img_name, np.round(pred_orig),
vmin=0, vmax=1, cmap='gray')
def test_app(config, model_path, data_path, best):
......@@ -52,7 +64,8 @@ def test_app(config, model_path, data_path, best):
"""
# init configs
session_config = ConfigProto(device_count={'GPU': 0})
run_config = tf.estimator.RunConfig(session_config=session_config, **config['tf']['run'])
run_config = tf.estimator.RunConfig(session_config=session_config,
**config['tf']['run'])
# find checkpoint
dir_run = model_path
......@@ -64,22 +77,28 @@ def test_app(config, model_path, data_path, best):
final_dir = dir_run + '/'
files = [f.path for f in os.scandir(final_dir)]
files = [x for x in files if 'ckpt' in x and x.endswith('.index')]
files = sorted(files, key=lambda x: int(x.split('.')[-2].split('-')[-1]), reverse=True) # sort chkpts
ckpt_to_initialize_from = '.'.join(files[0].split('.')[:-1]) # use the last one
files = sorted(files,
key=lambda x: int(x.split('.')[-2].split('-')[-1]),
reverse=True) # sort chkpts
ckpt_to_initialize_from = '.'.join(
files[0].split('.')[:-1]) # use the last one
# get mean and std
mean_std_path = [dir_run + '/mean.png', dir_run + '/std.png']
# build dataset
fnames = [f.path for f in os.scandir(data_path)]
img_fnames = [x[:-4] for x in fnames if x.endswith(".png") and not x.endswith("Mask.png")]
img_fnames = [x[:-4] for x in fnames if
x.endswith(".png") and not x.endswith("Mask.png")]
def test_input_fn():
return build_test_dataset(img_fnames, mean_std_path, config)
warm_start_from = tf.estimator.WarmStartSettings(ckpt_to_initialize_from=ckpt_to_initialize_from)
model = tf.estimator.Estimator(model_fn=model_fn, warm_start_from=warm_start_from, config=run_config, params=config)
warm_start_from = tf.estimator.WarmStartSettings(
ckpt_to_initialize_from=ckpt_to_initialize_from)
model = tf.estimator.Estimator(model_fn=model_fn,
warm_start_from=warm_start_from,
config=run_config, params=config)
results = model.predict(input_fn=test_input_fn)
save_predict_results(results=results, imgs=img_fnames, data_path=data_path)
# TODO calculate metrics if mask is available
"""
Parse parameters to test models
"""
import argparse
import logging
import os
......@@ -7,6 +11,11 @@ from yfmil3id2019.ui.test_app import test_app
def test(args=None):
"""
perform test
:param args:
:return:
"""
logging.getLogger("tensorflow").setLevel(logging.FATAL)
# parse args
......@@ -26,7 +35,8 @@ def test(args=None):
parser.add_argument('--best',
dest='best',
action='store_true',
help='use the best model instead of the final one, otherwise use the last checkpoint')
help='use the best model instead of the final one, '
'otherwise use the last checkpoint')
parser.add_argument('-b',
'--bs',
default=64,
......@@ -43,12 +53,13 @@ def test(args=None):
if data_path[-1] == '/':
data_path = data_path[:-1]
config_path = model_path + '/config_backup.yaml'
with open(config_path) as file:
config = yaml.load(file)
with open(config_path) as f:
config = yaml.load(f)
# modify config
config['model']['opt']['batch_size'] = args.bs
config['tf']['gpu'] = args.gpu
os.environ['CUDA_VISIBLE_DEVICES'] = config['tf']['gpu']
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
test_app(config=config, model_path=model_path, data_path=data_path, best=args.best)
test_app(config=config, model_path=model_path, data_path=data_path,
best=args.best)
# coding=utf-8
"""
Functions defined here are used for training
"""
import os
import tensorflow as tf
import yaml
from yfmil3id2019.src.cv import get_folder_name_from_paths, split_train_eval
from yfmil3id2019.src.data.load import build_dataset, get_labeled_folders, get_unlabeled_folders
from yfmil3id2019.src.data.load import (build_dataset, get_labeled_folders,
get_unlabeled_folders)
from yfmil3id2019.src.model.exporter import BestCheckpointCopier
from yfmil3id2019.src.model.model import model_fn
from yfmil3id2019.src.util import init_log_dir, make_dir, set_tf_logger, verify_dirs
from yfmil3id2019.src.util import (init_log_dir, make_dir, set_tf_logger,
verify_dirs)
from yfmil3id2019.src.wrapper.util import ConfigProto
def _train(run, config, dir_log):
"""
one single training for a cross validation fold
:param run:
:param config:
:param dir_log:
:return:
"""
# get run name, i.e. eval folder names
run_name = get_folder_name_from_paths(run.folders_lbl_eval)
# create cross validation folder
dir_run = dir_log + run_name
make_dir(dir_run)
# backup the config
with open(dir_run + '/config_backup.yaml', 'w') as f:
f.write(yaml.dump(config))
# calculate mean and std
mean_std_path = run.generate_mean_std(dir_run)
# input functions
def train_input_fn():
return build_dataset(folders_lbl=run.folders_lbl_train,
folders_unlbl=run.folders_unlbl_train,
mean_std_path=mean_std_path,
training=True,
config=config)
def eval_input_fn():
return build_dataset(folders_lbl=run.folders_lbl_eval,
folders_unlbl=None,
mean_std_path=mean_std_path,
training=False,
config=config)
# save best models
best_exporter = BestCheckpointCopier(
name='BestCheckpointCopier',
chkpt_dir=dir_run + '/export/best/',
metric_name=config['tf']['save_best']['metric_name'],
large_is_better=config['tf']['save_best']['large_is_better'])
# init configs
session_config = ConfigProto()
run_config = tf.estimator.RunConfig(session_config=session_config,
**config['tf']['run'])
train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn,
**config['tf']['train'])
eval_spec = tf.estimator.EvalSpec(input_fn=eval_input_fn,
exporters=best_exporter,
**config['tf']['eval'])
# train and eval
model = tf.estimator.Estimator(model_fn=model_fn,
model_dir=dir_run,
config=run_config,
params=config)
tf.estimator.train_and_evaluate(estimator=model,
train_spec=train_spec,
eval_spec=eval_spec)
def train_app(config, app_name):
"""
:param config:
:param app_name:
:return:
"""
# create log folder
cwd = os.getcwd() + '/'
dir_log = init_log_dir(cwd + config['dir']['log'], app_name)
......@@ -25,37 +97,4 @@ def train_app(config, app_name):
runs = split_train_eval(fs_lbl, fs_unlbl, config['data']['cv'])
for run in runs:
# get run name, i.e. eval folder names
run_name = get_folder_name_from_paths(run.folders_lbl_eval)
# create cross validation folder
dir_run = dir_log + run_name
make_dir(dir_run)
# backup the config
with open(dir_run + '/config_backup.yaml', 'w') as f:
f.write(yaml.dump(config))
# calculate mean and std
mean_std_path = run.generate_mean_std(dir_run)
# input functions
def train_input_fn():
return build_dataset(run.folders_lbl_train, run.folders_unlbl_train, mean_std_path, training=True, config=config)
def eval_input_fn():
return build_dataset(run.folders_lbl_eval, None, mean_std_path, training=False, config=config)
# save best models
best_exporter = BestCheckpointCopier(name='BestCheckpointCopier', chkpt_dir=dir_run + '/export/best/',
metric_name=config['tf']['save_best']['metric_name'], large_is_better=config['tf']['save_best']['large_is_better'])
# init configs
session_config = ConfigProto()
run_config = tf.estimator.RunConfig(session_config=session_config, **config['tf']['run'])
train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn, **config['tf']['train'])
eval_spec = tf.estimator.EvalSpec(input_fn=eval_input_fn, exporters=best_exporter, **config['tf']['eval'])
# train and eval
model = tf.estimator.Estimator(model_fn=model_fn, model_dir=dir_run, config=run_config, params=config)
tf.estimator.train_and_evaluate(estimator=model, train_spec=train_spec, eval_spec=eval_spec)
_train(run, config, dir_log)
# coding=utf-8
"""
Parse parameters to launch training
"""
import argparse
import os
......@@ -8,6 +10,11 @@ from yfmil3id2019.ui.train_app import train_app
def train(args=None):
"""
:param args:
:return:
"""
parser = argparse.ArgumentParser(description='yfmil3id2019_train')
parser.add_argument('-p',
......@@ -22,8 +29,8 @@ def train(args=None):
args = parser.parse_args(args)
# load config
with open(args.path) as file:
config = yaml.load(file)
with open(args.path) as f:
config = yaml.load(f)
# specify GPU
config['tf']['gpu'] = args.gpu
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment