Commit 3ae6173e authored by mathpluscode's avatar mathpluscode

ensemble last several checkpoints for final prediction, Fixed #1

parent e563d12d
Pipeline #3317 failed with stages
in 5 minutes and 38 seconds
...@@ -26,43 +26,64 @@ NUM_CPUS = cpu_count() ...@@ -26,43 +26,64 @@ NUM_CPUS = cpu_count()
ray.init(num_cpus=NUM_CPUS) ray.init(num_cpus=NUM_CPUS)
def save_predict_results(results, dir_run, name, preprocess): def save_predict_results(results, dir_pred, orig_size, ckpt_id):
""" """
:param results: :param results:
:param dir_run: :param dir_pred:
:param name:
:param preprocess: :param preprocess:
:return: :return:
""" """
dir_pred = dir_run + '/preds/%s/' % name
make_dir(dir_pred)
height = preprocess['shape']['orig'][0]
width = preprocess['shape']['orig'][1]
border_width = preprocess['shape']['border']
adjusted_width = width - 2 * border_width
orig_size = (adjusted_width, height)
preds = list(results) # iterator to list, to get the predictions preds = list(results) # iterator to list, to get the predictions
preds_bin = []
for sample_id, pred_dict in enumerate(preds): for sample_id, pred_dict in enumerate(preds):
# get predictions # get predictions
image = pred_dict['images'] image = pred_dict['images']
mask = pred_dict['masks'] mask = pred_dict['masks']
pred = pred_dict['preds'] pred = pred_dict['preds']
pred_orig = cv2.resize(pred, dsize=orig_size) pred_bin = np.round(pred)
pred_orig_bin = np.round(cv2.resize(pred, dsize=orig_size))
# output # output
image = (image - np.min(image)) / (np.max(image) - np.min(image)) image = (image - np.min(image)) / (np.max(image) - np.min(image))
imsave(dir_pred + '/%d_image.png' % sample_id, image) if ckpt_id == 0: # same image and mask once
imsave(dir_pred + '/%d_image.png' % sample_id,
image)
imsave(dir_pred + '/%d_mask.png' % sample_id, imsave(dir_pred + '/%d_mask.png' % sample_id,
mask, vmin=0, vmax=1, cmap='gray') mask, vmin=0, vmax=1, cmap='gray')
imsave(dir_pred + '/%d_prob.png' % sample_id, imsave(dir_pred + '/%d_prob_%d.png' % (sample_id, ckpt_id),
pred, vmin=0, vmax=1, cmap='gray') pred, vmin=0, vmax=1, cmap='gray')
imsave(dir_pred + '/%d_pred.png' % sample_id, imsave(dir_pred + '/%d_pred_%d.png' % (sample_id, ckpt_id),
np.round(pred), vmin=0, vmax=1, cmap='gray') pred_bin, vmin=0, vmax=1, cmap='gray')
imsave(dir_pred + '/%d_pred_orig.png' % sample_id, imsave(dir_pred + '/%d_pred_orig_%d.png' % (sample_id, ckpt_id),
np.round(pred_orig), vmin=0, vmax=1, cmap='gray') pred_orig_bin, vmin=0, vmax=1, cmap='gray')
preds_bin.append(pred_bin.astype(np.bool))
return preds_bin
def ensemble_predict(preds_all, dir_pred, orig_size):
"""
:param preds_all: contains N lists, each list corresponds to a chkpt
each list contains M images of shape (H, W) of type bool
:return:
"""
num_ckpts = len(preds_all)
stacked = np.stack([np.stack(preds, axis=0) for preds in preds_all],
axis=0) # shape = (N, M, H, W)
stacked = np.sum(stacked, axis=0) # shape = (M, H, W)
stacked = (stacked > (num_ckpts / 2)).astype(np.float)
for img_id in range(stacked.shape[0]):
# get predictions
pred_bin = stacked[img_id, :, :]
pred_orig_bin = np.round(cv2.resize(pred_bin, dsize=orig_size))
# output
imsave(dir_pred + '/%d_pred.png' % img_id, pred_bin, vmin=0, vmax=1,
cmap='gray')
imsave(dir_pred + '/%d_pred_orig.png' % img_id, pred_orig_bin, vmin=0,
vmax=1, cmap='gray')
@ray.remote @ray.remote
...@@ -141,8 +162,8 @@ def calculate_metrics(fnames, dir_cut, eval_train, return_hd): ...@@ -141,8 +162,8 @@ def calculate_metrics(fnames, dir_cut, eval_train, return_hd):
f.write(res) f.write(res)
def _eval(run, config, dir_run, best, def _eval(run, config, dir_run, best: bool, eval_train: bool, predict: bool,
eval_train: bool, predict: bool, return_hd: bool): return_hd: bool, num_ckpt: int):
""" """
evaluate for a model trained on one cross validation fold evaluate for a model trained on one cross validation fold
:param run: :param run:
...@@ -163,31 +184,34 @@ def _eval(run, config, dir_run, best, ...@@ -163,31 +184,34 @@ def _eval(run, config, dir_run, best,
run_config = tf.estimator.RunConfig(session_config=session_config, run_config = tf.estimator.RunConfig(session_config=session_config,
**config['tf']['run']) **config['tf']['run'])
# eval, generate predictions
if best: if best:
best_dir = dir_run + '/export/best/' chkpt_dir = dir_run + '/export/best/'
files = [f.path for f in os.scandir(best_dir)]
ckpt_to_initialize_from = '.'.join(files[0].split('.')[:-1])
save_name = 'best' save_name = 'best'
else: else:
final_dir = dir_run + '/' chkpt_dir = dir_run + '/'
files = [f.path for f in os.scandir(final_dir)]
files = [x for x in files if 'ckpt' in x and x.endswith('.index')]
files = sorted(files,
key=lambda x: int(x.split('.')[-2].split('-')[-1]),
reverse=True) # sort chkpts
ckpt_to_initialize_from = '.'.join(
files[0].split('.')[:-1]) # use the last one
save_name = 'final' save_name = 'final'
if eval_train: if eval_train:
save_name += '_all' save_name += '_all'
if predict: if predict:
# create folder
dir_pred = dir_run + '/preds/%s/' % save_name
make_dir(dir_pred)
# orig size
preprocess = config['data']['preprocess']
height = preprocess['shape']['orig'][0]
width = preprocess['shape']['orig'][1]
border_width = preprocess['shape']['border']
adjusted_width = width - 2 * border_width
orig_size = (adjusted_width, height)
# get mean and std # get mean and std
mean_std_path = [dir_run + '/mean.png', dir_run + '/std.png'] mean_std_path = [dir_run + '/mean.png', dir_run + '/std.png']
eval_folders = run.folders_lbl_eval eval_folders = run.folders_lbl_eval
# input data
def eval_input_fn(): def eval_input_fn():
return build_dataset(folders_lbl=eval_folders, return build_dataset(folders_lbl=eval_folders,
folders_unlbl=None, folders_unlbl=None,
...@@ -195,7 +219,24 @@ def _eval(run, config, dir_run, best, ...@@ -195,7 +219,24 @@ def _eval(run, config, dir_run, best,
training=False, training=False,
config=config) config=config)
print('evaluate for %s - %s' % (dir_run, save_name)) # find checkpoint files
files = [f.path for f in os.scandir(chkpt_dir)]
files = [x for x in files if 'ckpt' in x and x.endswith('.index')]
files = sorted(files,
key=lambda x: int(x.split('.')[-2].split('-')[-1]),
reverse=True) # sort chkpts, files[0] is the last one
ckpts_to_initialize_from = ['.'.join(f.split('.')[:-1]) for f in files]
print(len(ckpts_to_initialize_from), num_ckpt, type(num_ckpt))
ckpts_to_initialize_from = ckpts_to_initialize_from[:num_ckpt]
assert len(ckpts_to_initialize_from) % 2 == 1 # odd so they can vote
# load and evaluate
preds_all = []
for ckpt_id, ckpt_to_initialize_from in enumerate(
ckpts_to_initialize_from):
print('evaluate for %s - %s - %d-th chkpt' % (
dir_run, save_name, ckpt_id))
# predict
warm_start_from = tf.estimator.WarmStartSettings( warm_start_from = tf.estimator.WarmStartSettings(
ckpt_to_initialize_from=ckpt_to_initialize_from) ckpt_to_initialize_from=ckpt_to_initialize_from)
model = tf.estimator.Estimator(model_fn=model_fn, model = tf.estimator.Estimator(model_fn=model_fn,
...@@ -203,10 +244,16 @@ def _eval(run, config, dir_run, best, ...@@ -203,10 +244,16 @@ def _eval(run, config, dir_run, best,
config=run_config, config=run_config,
params=config) params=config)
results = model.predict(input_fn=eval_input_fn) results = model.predict(input_fn=eval_input_fn)
save_predict_results(results=results, # save and compress
dir_run=dir_run, preds = save_predict_results(
name=save_name, results=results,
preprocess=config['data']['preprocess']) dir_pred=dir_pred,
orig_size=orig_size,
ckpt_id=ckpt_id)
preds_all.append(preds)
# ensemble
ensemble_predict(preds_all=preds_all, dir_pred=dir_pred,
orig_size=orig_size) # ensemble the predictions
# calculate metrics # calculate metrics
print('calculate metric for %s - %s' % (dir_run, save_name)) print('calculate metric for %s - %s' % (dir_run, save_name))
...@@ -218,7 +265,7 @@ def _eval(run, config, dir_run, best, ...@@ -218,7 +265,7 @@ def _eval(run, config, dir_run, best,
return_hd=return_hd, eval_train=eval_train) return_hd=return_hd, eval_train=eval_train)
def eval_app(config, path, best, eval_train, predict, return_hd): def eval_app(config, path, best, eval_train, predict, return_hd, num_ckpt):
""" """
:param config: :param config:
:param path: :param path:
...@@ -240,4 +287,5 @@ def eval_app(config, path, best, eval_train, predict, return_hd): ...@@ -240,4 +287,5 @@ def eval_app(config, path, best, eval_train, predict, return_hd):
dir_run = path + '/' + run_name dir_run = path + '/' + run_name
if eval_train: if eval_train:
run.folders_lbl_eval = fs_lbl run.folders_lbl_eval = fs_lbl
_eval(run, config, dir_run, best, eval_train, predict, return_hd) _eval(run, config, dir_run, best, eval_train, predict, return_hd,
num_ckpt)
...@@ -49,6 +49,10 @@ def evaluate(args=None): ...@@ -49,6 +49,10 @@ def evaluate(args=None):
dest='return_hd', dest='return_hd',
action='store_true', action='store_true',
help='calculate hausdorff distance') help='calculate hausdorff distance')
parser.add_argument('-c',
'--num_ckpt',
default=1,
help='max number of checkpoint used')
parser.set_defaults(best=False) parser.set_defaults(best=False)
parser.set_defaults(eval=False) parser.set_defaults(eval=False)
parser.set_defaults(all=False) parser.set_defaults(all=False)
...@@ -71,4 +75,5 @@ def evaluate(args=None): ...@@ -71,4 +75,5 @@ def evaluate(args=None):
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true' os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
eval_app(config=config, path=path, best=args.best, eval_train=args.all, eval_app(config=config, path=path, best=args.best, eval_train=args.all,
predict=args.eval, return_hd=args.return_hd) predict=args.eval, return_hd=args.return_hd,
num_ckpt=int(args.num_ckpt))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment