Commit 3ae6173e authored by mathpluscode's avatar mathpluscode

ensemble last several checkpoints for final prediction, Fixed #1

parent e563d12d
Pipeline #3317 failed with stages
in 5 minutes and 38 seconds
......@@ -26,43 +26,64 @@ NUM_CPUS = cpu_count()
ray.init(num_cpus=NUM_CPUS)
def save_predict_results(results, dir_run, name, preprocess):
def save_predict_results(results, dir_pred, orig_size, ckpt_id):
"""
:param results:
:param dir_run:
:param name:
:param dir_pred:
:param preprocess:
:return:
"""
dir_pred = dir_run + '/preds/%s/' % name
make_dir(dir_pred)
height = preprocess['shape']['orig'][0]
width = preprocess['shape']['orig'][1]
border_width = preprocess['shape']['border']
adjusted_width = width - 2 * border_width
orig_size = (adjusted_width, height)
preds = list(results) # iterator to list, to get the predictions
preds_bin = []
for sample_id, pred_dict in enumerate(preds):
# get predictions
image = pred_dict['images']
mask = pred_dict['masks']
pred = pred_dict['preds']
pred_orig = cv2.resize(pred, dsize=orig_size)
pred_bin = np.round(pred)
pred_orig_bin = np.round(cv2.resize(pred, dsize=orig_size))
# output
image = (image - np.min(image)) / (np.max(image) - np.min(image))
imsave(dir_pred + '/%d_image.png' % sample_id, image)
if ckpt_id == 0: # same image and mask once
imsave(dir_pred + '/%d_image.png' % sample_id,
image)
imsave(dir_pred + '/%d_mask.png' % sample_id,
mask, vmin=0, vmax=1, cmap='gray')
imsave(dir_pred + '/%d_prob.png' % sample_id,
imsave(dir_pred + '/%d_prob_%d.png' % (sample_id, ckpt_id),
pred, vmin=0, vmax=1, cmap='gray')
imsave(dir_pred + '/%d_pred.png' % sample_id,
np.round(pred), vmin=0, vmax=1, cmap='gray')
imsave(dir_pred + '/%d_pred_orig.png' % sample_id,
np.round(pred_orig), vmin=0, vmax=1, cmap='gray')
imsave(dir_pred + '/%d_pred_%d.png' % (sample_id, ckpt_id),
pred_bin, vmin=0, vmax=1, cmap='gray')
imsave(dir_pred + '/%d_pred_orig_%d.png' % (sample_id, ckpt_id),
pred_orig_bin, vmin=0, vmax=1, cmap='gray')
preds_bin.append(pred_bin.astype(np.bool))
return preds_bin
def ensemble_predict(preds_all, dir_pred, orig_size):
"""
:param preds_all: contains N lists, each list corresponds to a chkpt
each list contains M images of shape (H, W) of type bool
:return:
"""
num_ckpts = len(preds_all)
stacked = np.stack([np.stack(preds, axis=0) for preds in preds_all],
axis=0) # shape = (N, M, H, W)
stacked = np.sum(stacked, axis=0) # shape = (M, H, W)
stacked = (stacked > (num_ckpts / 2)).astype(np.float)
for img_id in range(stacked.shape[0]):
# get predictions
pred_bin = stacked[img_id, :, :]
pred_orig_bin = np.round(cv2.resize(pred_bin, dsize=orig_size))
# output
imsave(dir_pred + '/%d_pred.png' % img_id, pred_bin, vmin=0, vmax=1,
cmap='gray')
imsave(dir_pred + '/%d_pred_orig.png' % img_id, pred_orig_bin, vmin=0,
vmax=1, cmap='gray')
@ray.remote
......@@ -141,8 +162,8 @@ def calculate_metrics(fnames, dir_cut, eval_train, return_hd):
f.write(res)
def _eval(run, config, dir_run, best,
eval_train: bool, predict: bool, return_hd: bool):
def _eval(run, config, dir_run, best: bool, eval_train: bool, predict: bool,
return_hd: bool, num_ckpt: int):
"""
evaluate for a model trained on one cross validation fold
:param run:
......@@ -163,31 +184,34 @@ def _eval(run, config, dir_run, best,
run_config = tf.estimator.RunConfig(session_config=session_config,
**config['tf']['run'])
# eval, generate predictions
if best:
best_dir = dir_run + '/export/best/'
files = [f.path for f in os.scandir(best_dir)]
ckpt_to_initialize_from = '.'.join(files[0].split('.')[:-1])
chkpt_dir = dir_run + '/export/best/'
save_name = 'best'
else:
final_dir = dir_run + '/'
files = [f.path for f in os.scandir(final_dir)]
files = [x for x in files if 'ckpt' in x and x.endswith('.index')]
files = sorted(files,
key=lambda x: int(x.split('.')[-2].split('-')[-1]),
reverse=True) # sort chkpts
ckpt_to_initialize_from = '.'.join(
files[0].split('.')[:-1]) # use the last one
chkpt_dir = dir_run + '/'
save_name = 'final'
if eval_train:
save_name += '_all'
if predict:
# create folder
dir_pred = dir_run + '/preds/%s/' % save_name
make_dir(dir_pred)
# orig size
preprocess = config['data']['preprocess']
height = preprocess['shape']['orig'][0]
width = preprocess['shape']['orig'][1]
border_width = preprocess['shape']['border']
adjusted_width = width - 2 * border_width
orig_size = (adjusted_width, height)
# get mean and std
mean_std_path = [dir_run + '/mean.png', dir_run + '/std.png']
eval_folders = run.folders_lbl_eval
# input data
def eval_input_fn():
return build_dataset(folders_lbl=eval_folders,
folders_unlbl=None,
......@@ -195,7 +219,24 @@ def _eval(run, config, dir_run, best,
training=False,
config=config)
print('evaluate for %s - %s' % (dir_run, save_name))
# find checkpoint files
files = [f.path for f in os.scandir(chkpt_dir)]
files = [x for x in files if 'ckpt' in x and x.endswith('.index')]
files = sorted(files,
key=lambda x: int(x.split('.')[-2].split('-')[-1]),
reverse=True) # sort chkpts, files[0] is the last one
ckpts_to_initialize_from = ['.'.join(f.split('.')[:-1]) for f in files]
print(len(ckpts_to_initialize_from), num_ckpt, type(num_ckpt))
ckpts_to_initialize_from = ckpts_to_initialize_from[:num_ckpt]
assert len(ckpts_to_initialize_from) % 2 == 1 # odd so they can vote
# load and evaluate
preds_all = []
for ckpt_id, ckpt_to_initialize_from in enumerate(
ckpts_to_initialize_from):
print('evaluate for %s - %s - %d-th chkpt' % (
dir_run, save_name, ckpt_id))
# predict
warm_start_from = tf.estimator.WarmStartSettings(
ckpt_to_initialize_from=ckpt_to_initialize_from)
model = tf.estimator.Estimator(model_fn=model_fn,
......@@ -203,10 +244,16 @@ def _eval(run, config, dir_run, best,
config=run_config,
params=config)
results = model.predict(input_fn=eval_input_fn)
save_predict_results(results=results,
dir_run=dir_run,
name=save_name,
preprocess=config['data']['preprocess'])
# save and compress
preds = save_predict_results(
results=results,
dir_pred=dir_pred,
orig_size=orig_size,
ckpt_id=ckpt_id)
preds_all.append(preds)
# ensemble
ensemble_predict(preds_all=preds_all, dir_pred=dir_pred,
orig_size=orig_size) # ensemble the predictions
# calculate metrics
print('calculate metric for %s - %s' % (dir_run, save_name))
......@@ -218,7 +265,7 @@ def _eval(run, config, dir_run, best,
return_hd=return_hd, eval_train=eval_train)
def eval_app(config, path, best, eval_train, predict, return_hd):
def eval_app(config, path, best, eval_train, predict, return_hd, num_ckpt):
"""
:param config:
:param path:
......@@ -240,4 +287,5 @@ def eval_app(config, path, best, eval_train, predict, return_hd):
dir_run = path + '/' + run_name
if eval_train:
run.folders_lbl_eval = fs_lbl
_eval(run, config, dir_run, best, eval_train, predict, return_hd)
_eval(run, config, dir_run, best, eval_train, predict, return_hd,
num_ckpt)
......@@ -49,6 +49,10 @@ def evaluate(args=None):
dest='return_hd',
action='store_true',
help='calculate hausdorff distance')
parser.add_argument('-c',
'--num_ckpt',
default=1,
help='max number of checkpoint used')
parser.set_defaults(best=False)
parser.set_defaults(eval=False)
parser.set_defaults(all=False)
......@@ -71,4 +75,5 @@ def evaluate(args=None):
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
eval_app(config=config, path=path, best=args.best, eval_train=args.all,
predict=args.eval, return_hd=args.return_hd)
predict=args.eval, return_hd=args.return_hd,
num_ckpt=int(args.num_ckpt))
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment