Commit 54d33344 authored by mathpluscode's avatar mathpluscode

stats and curve pipeline works

parent 845a5113
......@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
import sys
from yfmil3id2019.ui.yfmil3id2019_test_command_line import main
from yfmil3id2019.ui.analyse_command_line import analyse
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
sys.exit(analyse(sys.argv[1:]))
......@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
import sys
from yfmil3id2019.ui.train_command_line import main
from yfmil3id2019.ui.train_command_line import train
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
sys.exit(train(sys.argv[1:]))
......@@ -42,10 +42,10 @@ def form_msg(values, name):
return msg
def read_folder(dir_path, best, total, evalAll):
def read_folder(dir_path, best, all, total):
folder_name = 'best' if best else 'final'
if evalAll:
folder_name += 'All'
if all:
folder_name += '_all'
log_fname = '/preds/%s/metric.log' % folder_name
folder_paths = [f.path for f in os.scandir(dir_path) if f.is_dir()]
folder_paths = [x for x in folder_paths if os.path.exists(x + log_fname)]
......@@ -77,8 +77,8 @@ def read_folder(dir_path, best, total, evalAll):
return metrics_total, metrics_sep, msgs
def read_folder_evalAll(dir_path, best, total):
log_fname = '/preds/%s/metric.log' % ('bestAll' if best else 'finalAll')
def read_folder_all(dir_path, best):
log_fname = '/preds/%s/metric.log' % ('best_all' if best else 'final_all')
metrics = []
for folder in PATIENT_FOLDER_NAMES:
......
import os
import numpy as np
from scipy.stats import wilcoxon
from _plotly_future_ import v4_subplots
import plotly.graph_objs as go
from plotly.offline import plot
from plotly.colors import DEFAULT_PLOTLY_COLORS
import plotly.io as pio
from plotly.subplots import make_subplots
from yfmil3id2019.src.eval import read_folder, read_folder_all
def analyse_stats(log_path, best, all, total=False):
"""
analyse all experiments under log path
:param log_path:
:param best: if using the best chkpt
:param all: if analyse both train and test
:param total: if present stats on each patient
:return:
"""
folder_paths = [f.path for f in os.scandir(log_path) if f.is_dir()]
# print stats of metrics
names = []
msgs = dict()
metrics_total = dict()
metrics_sep = dict()
for p in folder_paths:
name = ''.join(p.split('-')[1:])
m_total, m_sep, msg = read_folder(p, best=best, total=total, all=all)
names.append(name)
metrics_total[name] = m_total
metrics_sep[name] = m_sep
msgs[name] = msg
for n in names:
print(msgs[n])
if all:
print('WARNING: stats are calculated on both train and test!')
# example of wilcoxon test
if len(names) > 1:
try:
print('wilcoxon test between %s and %s: p-value = %e'
% (names[0], names[1], wilcoxon(metrics_total[names[0]]['f1'], metrics_total[names[1]]['f1']).pvalue))
except:
pass
def rgb2rgba(color, opacity):
# rgb(31, 119, 180) -> rgba(31, 119, 180, 0.2)
return 'rgba' + color[3:-1] + ',%.1f)' % opacity
def analyse_foreground(log_path, best, std):
folder_paths = [f.path for f in os.scandir(log_path) if f.is_dir()]
# stats of metrics
names = []
metrics_total = dict()
for p in folder_paths:
name = ''.join(p.split('-')[1:])
m_total, _, _ = read_folder(p, best=best, total=False)
names.append(name)
metrics_total[name] = m_total
# plot the foreground proportion vs performance curve
row = col = 1
num_bins = 30
for metric_name in ['f1', 'hd_p95']:
if metric_name not in metrics_total[names[0]].keys():
print('metric %s not found' % metric_name)
continue
fig = make_subplots(rows=1, cols=1)
for i, n in enumerate(names):
metrics = metrics_total[n]
props = metrics['prop']
vals = metrics[metric_name]
bin_size = (len(props) // num_bins) + 1
pairs = sorted(zip(props, vals), key=lambda x: x[0])
bins = [pairs[(bin_size * i):(bin_size * i + bin_size)] for i in range(num_bins)]
prop_mean = [np.mean([x[0] for x in bin]) for bin in bins]
val_mean = np.array([np.mean([x[1] for x in bin]) for bin in bins])
val_std = np.array([np.std([x[1] for x in bin]) for bin in bins])
fig.append_trace(go.Scatter(x=prop_mean, y=val_mean,
mode='lines+markers',
marker=dict(color=DEFAULT_PLOTLY_COLORS[i % 10]),
showlegend=False,
name=n,
),
row=row, col=col)
if std:
fig.add_trace(go.Scatter(x=prop_mean, y=val_mean - val_std,
fill=None,
mode='lines',
line_color=rgb2rgba(DEFAULT_PLOTLY_COLORS[col - 1], 0.4),
showlegend=False, name='mean-std'),
row=row, col=col)
fig.add_trace(go.Scatter(x=prop_mean, y=val_mean + val_std,
fill='tonexty',
mode='lines',
line_color=rgb2rgba(DEFAULT_PLOTLY_COLORS[col - 1], 0.4),
showlegend=False, name='mean+std'),
row=row, col=col)
fig.update_xaxes(range=[0, 1], title=dict(text='Liver Proportion'), row=row, col=col)
fig.layout.update(autosize=False,
width=800,
height=800)
plot(fig, filename='%s_vs_prop.html' % metric_name, auto_open=True)
pio.write_image(fig, '%s_vs_prop.pdf' % metric_name)
def analyse_foreground_all(log_path, best, std):
folder_paths = [f.path for f in os.scandir(log_path) if f.is_dir()]
# stats of metrics
names = []
metrics_sep = dict()
for p in folder_paths:
name = ''.join(p.split('-')[1:])
m_sep = read_folder_all(p, best=best)
names.append(name)
metrics_sep[name] = m_sep
# plot the foreground proportion vs performance curve
row = col = 1
num_bins = 30
for metric_name in ['f1', 'hd_p95']:
if metric_name not in metrics_sep[names[0]].keys():
print('metric %s not found' % metric_name)
continue
fig = make_subplots(rows=1, cols=1)
for mode in ['test', 'train']:
for i, n in enumerate(names):
metrics = metrics_sep[n]
props = metrics['prop']['test']
vals = metrics[metric_name][mode]
if mode == 'train':
vals = np.mean(vals, axis=0)
bin_size = (len(props) // num_bins) + 1
pairs = sorted(zip(props, vals), key=lambda x: x[0])
bins = [pairs[(bin_size * i):(bin_size * i + bin_size)] for i in range(num_bins)]
prop_mean = [np.mean([x[0] for x in bin]) for bin in bins]
val_mean = np.array([np.mean([x[1] for x in bin]) for bin in bins])
val_std = np.array([np.std([x[1] for x in bin]) for bin in bins])
fig.append_trace(go.Scatter(x=prop_mean, y=val_mean,
mode='lines+markers',
marker=dict(color=DEFAULT_PLOTLY_COLORS[i % 10]),
showlegend=False,
name=n + '_' + mode,
),
row=row, col=col)
if std:
fig.add_trace(go.Scatter(x=prop_mean, y=val_mean - val_std,
fill=None,
mode='lines',
line_color=rgb2rgba(DEFAULT_PLOTLY_COLORS[col - 1], 0.4),
showlegend=False, name='mean-std'),
row=row, col=col)
fig.add_trace(go.Scatter(x=prop_mean, y=val_mean + val_std,
fill='tonexty',
mode='lines',
line_color=rgb2rgba(DEFAULT_PLOTLY_COLORS[col - 1], 0.4),
showlegend=False, name='mean+std'),
row=row, col=col)
fig.update_xaxes(range=[0, 1], title=dict(text='Liver Proportion'), row=row, col=col)
fig.layout.update(autosize=False,
width=800,
height=800)
plot(fig, filename='%s_vs_prop_train_test.html' % metric_name, auto_open=True)
pio.write_image(fig, '%s_vs_prop_train_test.pdf' % metric_name)
# coding=utf-8
import argparse
import os
import yaml
from yfmil3id2019.ui.analyse_app import analyse_stats, analyse_foreground, analyse_foreground_all
def analyse(args=None):
parser = argparse.ArgumentParser(description='yfmil3id2019_analyse')
parser.add_argument('-p',
'--path',
required=True,
help='Path of log dir')
parser.add_argument('--best',
dest='best',
action='store_true',
help='use the best model instead of the final one, otherwise use the last checkpoint')
parser.add_argument('--all',
dest='all',
action='store_true',
help='evaluate models on all data')
parser.add_argument('--std',
dest='std',
action='store_true',
help='plot std in foreground curve')
parser.set_defaults(best=False)
parser.set_defaults(all=False)
parser.set_defaults(std=False)
args = parser.parse_args(args)
analyse_stats(log_path=args.path, best=args.best, all=args.all)
if not args.all:
analyse_foreground(log_path=args.path, best=args.best, std=args.std)
else:
analyse_foreground_all(log_path=args.path, best=args.best, std=args.std)
......@@ -7,7 +7,7 @@ import yaml
from yfmil3id2019.ui.train_app import train_app
def main(args=None):
def train(args=None):
parser = argparse.ArgumentParser(description='yfmil3id2019_train')
parser.add_argument('-p',
......
# coding=utf-8
""" Put a useful module description here. Import whatever you want. """
import six
def run_app(network, weights, image, output, verbose):
""" Run the application, using command line args. """
six.print_("Network argument=" + str(network))
six.print_("Weights argument=" + str(weights))
six.print_("Image argument=" + str(image))
six.print_("Output argument=" + str(output))
six.print_("Verbose argument=" + str(verbose))
# coding=utf-8
"""Put a useful module description here. """
import argparse
from yfmil3id2019 import __version__
from yfmil3id2019.ui.yfmil3id2019_test_app import run_app
def main(args=None):
"""Entry point for yfmil3id2019_test application"""
parser = argparse.ArgumentParser(description='yfmil3id2019_test')
## ADD POSITIONAL ARGUMENTS
parser.add_argument("-w", "--weights",
required=True,
type=str,
help="Weights")
parser.add_argument("-n", "--network",
required=True,
type=str,
help="Network Architecture")
parser.add_argument("-i", "--image",
required=True,
type=str,
help="Image to test")
# ADD OPTINAL ARGUMENTS
parser.add_argument("-o", "--output",
required=True,
type=str,
help="Output file"
)
parser.add_argument("-v", "--verbose",
action="store_true",
help="Enable verbose output",
)
version_string = __version__
friendly_version_string = version_string if version_string else 'unknown'
parser.add_argument(
"--version",
action='version',
version='yfmil3id2019_test version ' + friendly_version_string)
args = parser.parse_args(args)
run_app(args.network, args.weights, args.image, args.output, args.verbose)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment