"""
File: app/tutorial_metrics.py
Project: 22HLT01 QUMPHY
Contact: nando.hegemann@ptb.de
Gitlab: https://gitlab.com/qumphy
Description: Tutorial for evaluation metrics for model performance.
"""
import numpy as np
from qumphy.metrics import auc_score_binary as auc_b
from qumphy.metrics import auc_score_multiclass as auc_mc
from qumphy.metrics import balanced_accuracy_score as acc_b
from qumphy.metrics import f1_score
from qumphy.metrics import matthews_correlation_coefficient as mcc
from qumphy.metrics import mean_absolute_error as mae
from qumphy.metrics import precision_score as ppv
from qumphy.metrics import root_mean_square_error as rmse
[docs]
def mc2ml(arr: np.ndarray, noise: float = 0.0) -> np.ndarray:
"""Helper function to convert multi-class array to multi-lable array.
Converts a 1D array with n classes in a matrix with n columns. Each row is zero
except for the column corresponding to the class lable, which is set to one.
Noise add a perturbation to the values and subsequently normalizes the rows
to mimic softmax values.
Parameters
----------
arr : np.ndarray
Array with two or more different values.
noise : float, optional
Noise level.
Returns
-------
np.ndarray
Multi-lable version of the array.
"""
assert arr.ndim == 1
n_cols = int(np.max(arr) - np.min(arr)) + 1
ret = np.zeros((arr.size, n_cols))
for j in range(n_cols):
idx = np.where(arr == j)[0]
ret[idx, j] = 1
if noise > 0:
# add noise and normalize to simulate softmax
ret += np.random.uniform(0, noise, ret.shape)
ret /= np.expand_dims(np.sum(ret, axis=1), 1)
return ret
if __name__ == "__main__":
# regression
# NOTE: Could be determination of systolic/diastolic blood pressure
print("regression:")
target = np.random.uniform(0, 1, (1000, 2))
eps = np.random.multivariate_normal(np.zeros(2), 1e-04 * np.eye(2), (1000,))
prediction = target + eps
print(f" MAE: {mae(target, prediction)}")
print(f" RMSE: {rmse(target, prediction)}")
# binary classification
# NOTE: Could be determination of AF (yes / no)
print()
print("binary classification:")
target = np.random.randint(0, 2, (1000,))
prediction = np.random.randint(0, 2, (1000,))
print(f" Acc_b: {acc_b(target, prediction)}")
print(f" AUC: {auc_b(target, prediction)}")
print(f" F1: {f1_score(target, prediction)}")
print(f" Mcc: {mcc(target, prediction)}")
print(f" PPV: {ppv(target, prediction)}")
# multi-class and multi-lable classification
# NOTE: Could be determination of hypertension classes:
# 1. normal
# 2. elevated
# 3. hypertension stage 1
# 4. hypertension stage 2
# 5. hypertensive crisis
target = np.random.randint(0, 5, (1000,))
prediction = np.random.randint(0, 5, (1000,))
pred_softmax = mc2ml(prediction, noise=1.0) # mock softmax classification
print()
print("multi-class classification:")
print(f" Acc_b: {acc_b(target, prediction)}")
print(f" F1(no-avg): {f1_score(target, prediction)}")
print(f" F1(micro): {f1_score(target, prediction, 'micro')}")
print(f" F1(macro): {f1_score(target, prediction, 'macro')}")
print(f" F1(weighted): {f1_score(target, prediction, 'weighted')}")
print(f" Mcc: {mcc(target, prediction)}")
print(f" PPV(no-avg): {ppv(target, prediction)}")
print(f" PPV(micro): {ppv(target, prediction, 'micro')}")
print(f" PPV(macro): {ppv(target, prediction, 'macro')}")
print(f" PPV(weighted): {ppv(target, prediction, 'weighted')}")
print()
print("multi-lable classification:")
print(f" AUC(ovr): {auc_mc(target, mc2ml(prediction), 'ovr')}")
print(f" AUC(ovo): {auc_mc(target, mc2ml(prediction), 'ovo')}")
print(f" AUC(ovr, softmax): {auc_mc(target, pred_softmax, 'ovr')}")
print(f" AUC(ovo, softmax): {auc_mc(target, pred_softmax, 'ovo')}")
print(f" F1(no-avg): {f1_score(mc2ml(target), mc2ml(prediction))}")
print(f" F1(micro): {f1_score(mc2ml(target), mc2ml(prediction), 'micro')}")
print(f" F1(macro): {f1_score(mc2ml(target), mc2ml(prediction), 'macro')}")
print(f" F1(weighted): {f1_score(mc2ml(target), mc2ml(prediction), 'weighted')}")
print(f" PPV(no-avg): {ppv(target, prediction)}")
print(f" PPV(micro): {ppv(target, prediction, 'micro')}")
print(f" PPV(macro): {ppv(target, prediction, 'macro')}")
print(f" PPV(weighted): {ppv(target, prediction, 'weighted')}")