diff --git a/EvaluateModel.ipynb b/EvaluateModel.ipynb index ac59e7b..5db9c16 100644 --- a/EvaluateModel.ipynb +++ b/EvaluateModel.ipynb @@ -19,23 +19,37 @@ "import pandas as pd\n", "from glob import glob\n", "import os\n", + "import random\n", + "import torch\n", "\n", "from actinet import __version__ as actinet_version\n", "from actinet.models import ActivityClassifier, RFActivityClassifier\n", - "from actinet.prepare import load_all_and_make_windows, extract_accelerometer_features, \\\n", - " prepare_accelerometer_data\n", + "from actinet.prepare import (\n", + " load_all_and_make_windows,\n", + " extract_accelerometer_features,\n", + " prepare_accelerometer_data,\n", + ")\n", "from actinet.evaluate import evaluate_models\n", "from actinet.utils.model_config import MODEL_CONFIG\n", - "from actinet.utils.eval_utils import calculate_metrics, DivDict, convert_version, \\\n", - " build_mae_table\n", - "from actinet.utils.plot_utils import plot_model_performance, plot_boxplots_panel, \\\n", - " generate_confusion_matrices, generate_bland_altman_plots, generate_bland_altman_panel, \\\n", - " plot_errors\n", - " \n", - "WINSEC = 30 # seconds\n", - "SAMPLE_RATE = 100 # Hz\n", - "RESAMPLE_RATE = 30 # Hz\n", - "N_JOBS = 8 # Set to higher number for quicker execution, but don't exceed max.\n", + "from actinet.utils.eval_utils import (\n", + " convert_version,\n", + " build_mae_table,\n", + " calculate_metrics,\n", + " DivDict,\n", + ")\n", + "from actinet.utils.plot_utils import (\n", + " plot_model_performance,\n", + " plot_boxplots_panel,\n", + " generate_confusion_matrices,\n", + " generate_bland_altman_plots,\n", + " generate_bland_altman_panel,\n", + " plot_errors,\n", + ")\n", + "\n", + "WINSEC = 30 # seconds\n", + "SAMPLE_RATE = 100 # Hz\n", + "RESAMPLE_RATE = 30 # Hz\n", + "N_JOBS = 8 # Set to higher number for quicker execution, but don't exceed max.\n", "DATAFILES = f\"data/capture24/P[0-9][0-9][0-9].csv.gz\"\n", "ANNOFILE = f\"data/capture24/annotation-label-dictionary.csv\"\n", "SAVEFOLDER = f\"data/capture24\"\n", @@ -43,7 +57,30 @@ "\n", "ACTIVITY_LABELS = list(MODEL_CONFIG[ANNOLABEL][\"labels\"].keys())\n", "\n", - "out_dir = f\"outputs/evaluation/{ANNOLABEL}/actinet_bbaa_{convert_version(actinet_version)}\"" + "out_dir = (\n", + " f\"outputs/evaluation/{ANNOLABEL}/actinet_bbaa_{convert_version(actinet_version)}\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def seed_evaluation(seed: int = 42):\n", + " os.environ[\"PYTHONHASHSEED\"] = str(seed)\n", + " os.environ[\"CUBLAS_WORKSPACE_CONFIG\"] = \":4096:8\"\n", + " random.seed(seed)\n", + " np.random.seed(seed)\n", + " torch.manual_seed(seed)\n", + " torch.cuda.manual_seed_all(seed)\n", + " torch.use_deterministic_algorithms(True)\n", + " torch.backends.cudnn.deterministic = True\n", + " torch.backends.cudnn.benchmark = False\n", + "\n", + "\n", + "seed_evaluation(42)" ] }, { @@ -91,7 +128,9 @@ " T_bbaa = np.load(f\"{bbaa_prepared_folder}/T.npy\")\n", " P_bbaa = np.load(f\"{bbaa_prepared_folder}/pid.npy\")\n", "else:\n", - " X_bbaa, Y_bbaa, T_bbaa, P_bbaa = prepare_accelerometer_data(ANNOFILE, ANNOLABEL, bbaa_prepared_folder, N_JOBS)" + " X_bbaa, Y_bbaa, T_bbaa, P_bbaa = prepare_accelerometer_data(\n", + " ANNOFILE, ANNOLABEL, bbaa_prepared_folder, N_JOBS\n", + " )" ] }, { @@ -110,9 +149,9 @@ "\n", "else:\n", " X_actinet, Y_actinet, T_actinet, P_actinet = load_all_and_make_windows(\n", - " datafiles=glob(DATAFILES), \n", - " annofile=ANNOFILE, \n", - " out_dir=actinet_prepared_folder, \n", + " datafiles=glob(DATAFILES),\n", + " annofile=ANNOFILE,\n", + " out_dir=actinet_prepared_folder,\n", " anno_label=ANNOLABEL,\n", " sample_rate=SAMPLE_RATE,\n", " winsec=WINSEC,\n", @@ -140,17 +179,17 @@ "bbaa_res_path = out_dir + \"/rf_results.pkl\"\n", "\n", "if not os.path.exists(actinet_res_path) or not os.path.exists(bbaa_res_path):\n", - " os.makedirs(out_dir+'/models', exist_ok=True)\n", + " os.makedirs(out_dir + \"/models\", exist_ok=True)\n", "\n", " rf_params = {\n", " **{\n", " \"winsec\": WINSEC,\n", " \"labels\": ACTIVITY_LABELS,\n", - " \"hmm_handle_sleep_transitions\": False,\n", + " \"hmm_handle_sleep_transitions\": True,\n", " \"hmm_ignore_transition_gaps\": False,\n", - " \"verbose\": 1\n", + " \"verbose\": 1,\n", " },\n", - " **MODEL_CONFIG[ANNOLABEL][\"rf_params\"]\n", + " **MODEL_CONFIG[ANNOLABEL][\"rf_params\"],\n", " }\n", "\n", " actinet_params = {\n", @@ -158,9 +197,9 @@ " \"labels\": ACTIVITY_LABELS,\n", " \"batch_size\": 1000,\n", " \"device\": \"cuda:0\",\n", - " \"verbose\": True\n", + " \"verbose\": True,\n", " },\n", - " **MODEL_CONFIG[ANNOLABEL][\"actinet_params\"]\n", + " **MODEL_CONFIG[ANNOLABEL][\"actinet_params\"],\n", " }\n", "\n", " eval_params = {\n", @@ -174,9 +213,9 @@ " \"datafiles\": DATAFILES,\n", " \"annofile\": ANNOFILE,\n", " \"savefolder\": SAVEFOLDER,\n", - " \"anno_label\": ANNOLABEL\n", + " \"anno_label\": ANNOLABEL,\n", " },\n", - " \"actinet_params\": actinet_params\n", + " \"actinet_params\": actinet_params,\n", " }\n", "\n", " bbaa_classifier = RFActivityClassifier(**rf_params)\n", @@ -194,7 +233,7 @@ " P_bbaa,\n", " T_actinet,\n", " T_bbaa,\n", - " weights_path=out_dir+\"/models/actinet_fold_{}.pt\",\n", + " weights_path=out_dir + \"/models/actinet_fold_{}.pt\",\n", " out_dir=out_dir,\n", " verbose=True,\n", " )\n", @@ -209,8 +248,13 @@ "metadata": {}, "outputs": [], "source": [ - "fold_pid_df = pd.DataFrame([{'Fold': fold+1, \"Test Participant IDs\": \", \".join(sorted(set(group)))} for fold, group in results_actinet[\"group\"].items()]).set_index(\"Fold\")\n", - "fold_pid_df.to_csv(\"outputs/actinet_vs_bbaa/fold_pids.csv\")\n", + "fold_pid_df = pd.DataFrame(\n", + " [\n", + " {\"Fold\": fold + 1, \"Test Participant IDs\": \", \".join(sorted(set(group)))}\n", + " for fold, group in results_actinet[\"group\"].items()\n", + " ]\n", + ").set_index(\"Fold\")\n", + "fold_pid_df.to_csv(f\"{out_dir}/fold_pids.csv\")\n", "fold_pid_df" ] }, @@ -221,14 +265,16 @@ "outputs": [], "source": [ "data = {\n", - " 'accelerometer': {'y': np.hstack(results_bbaa[\"Y_true\"]), \n", - " 'y_pred': np.hstack(results_bbaa[\"Y_pred\"]), \n", - " 'pid': np.hstack(results_bbaa[\"group\"])\n", - " },\n", - " 'actinet': {'y': np.hstack(results_actinet[\"Y_true\"]), \n", - " 'y_pred': np.hstack(results_actinet[\"Y_pred\"]), \n", - " 'pid': np.hstack(results_actinet[\"group\"])\n", - " }\n", + " \"accelerometer\": {\n", + " \"y\": np.hstack(results_bbaa[\"Y_true\"]),\n", + " \"y_pred\": np.hstack(results_bbaa[\"Y_pred\"]),\n", + " \"pid\": np.hstack(results_bbaa[\"group\"]),\n", + " },\n", + " \"actinet\": {\n", + " \"y\": np.hstack(results_actinet[\"Y_true\"]),\n", + " \"y_pred\": np.hstack(results_actinet[\"Y_pred\"]),\n", + " \"pid\": np.hstack(results_actinet[\"group\"]),\n", + " },\n", "}" ] }, @@ -263,11 +309,9 @@ "outputs": [], "source": [ "# Group by model and calculate mean and standard deviation\n", - "summary = results.groupby('Model')[['Accuracy',\n", - " 'Balanced Accuracy',\n", - " 'Cohen Kappa',\n", - " 'Macro F1']].agg(lambda x: f\"{np.mean(x):.3f} \" + \n", - " f\"± {np.std(x):.3f}\")\n", + "summary = results.groupby(\"Model\")[\n", + " [\"Accuracy\", \"Balanced Accuracy\", \"Cohen Kappa\", \"Macro F1\"]\n", + "].agg(lambda x: f\"{np.mean(x):.3f} \" + f\"± {np.std(x):.3f}\")\n", "\n", "summary" ] @@ -281,12 +325,14 @@ "metadata = pd.read_csv(\"data/capture24/metadata.csv\")\n", "results_df = results.merge(metadata, left_on=\"Participant\", right_on=\"pid\")\n", "\n", - "sex_mapping = {'F': 'Female', 'M': 'Male'}\n", - "results_df['Sex'] = pd.Categorical(results_df['sex'].map(sex_mapping), ordered=True,\n", - " categories=sex_mapping.values())\n", + "sex_mapping = {\"F\": \"Female\", \"M\": \"Male\"}\n", + "results_df[\"Sex\"] = pd.Categorical(\n", + " results_df[\"sex\"].map(sex_mapping), ordered=True, categories=sex_mapping.values()\n", + ")\n", "\n", - "results_df['Age Band'] = pd.Categorical(results_df['age'], ordered=True,\n", - " categories=['18-29', '30-37', '38-52', '53+'])\n", + "results_df[\"Age Band\"] = pd.Categorical(\n", + " results_df[\"age\"], ordered=True, categories=[\"18-29\", \"30-37\", \"38-52\", \"53+\"]\n", + ")\n", "\n", "results_df.drop(columns=[\"age\", \"sex\", \"pid\"], inplace=True)" ] @@ -306,12 +352,12 @@ "source": [ "from scipy.stats import ttest_rel\n", "\n", - "metrics = ['Accuracy', 'Balanced Accuracy', 'Cohen Kappa', 'Macro F1']\n", + "metrics = [\"Accuracy\", \"Balanced Accuracy\", \"Cohen Kappa\", \"Macro F1\"]\n", "p_values = {}\n", "\n", "for metric in metrics:\n", - " acc_values = results_df.loc[results_df['Model'] == 'accelerometer', metric].values\n", - " actinet_values = results_df.loc[results_df['Model'] == 'actinet', metric].values\n", + " acc_values = results_df.loc[results_df[\"Model\"] == \"accelerometer\", metric].values\n", + " actinet_values = results_df.loc[results_df[\"Model\"] == \"actinet\", metric].values\n", " _, p_value = ttest_rel(acc_values, actinet_values)\n", " p_values[metric] = p_value\n", "\n", @@ -324,7 +370,7 @@ "metadata": {}, "outputs": [], "source": [ - "plot_model_performance(results, 'Macro F1', 10)" + "plot_model_performance(results, \"Macro F1\", 10)" ] }, { @@ -333,8 +379,13 @@ "metadata": {}, "outputs": [], "source": [ - "plot_boxplots_panel(results_df, y=\"Macro F1\", by=[\"Age Band\", \"Sex\"], fontsize=12, \n", - " save_path=f\"{out_dir}/plots/boxplots_panel.png\")" + "plot_boxplots_panel(\n", + " results_df,\n", + " y=\"Macro F1\",\n", + " by=[\"Age Band\", \"Sex\"],\n", + " fontsize=12,\n", + " save_path=f\"{out_dir}/plots/boxplots_panel.png\",\n", + ")" ] }, { @@ -343,9 +394,9 @@ "metadata": {}, "outputs": [], "source": [ - "results_df.groupby(['Age Band', 'Model'])[['Cohen Kappa', 'Accuracy', 'Macro F1', 'Balanced Accuracy']].agg(\n", - " lambda x: f\"{np.mean(x):.3f} ± {np.std(x):.3f}\"\n", - ")" + "results_df.groupby([\"Age Band\", \"Model\"])[\n", + " [\"Cohen Kappa\", \"Accuracy\", \"Macro F1\", \"Balanced Accuracy\"]\n", + "].agg(lambda x: f\"{np.mean(x):.3f} ± {np.std(x):.3f}\")" ] }, { @@ -354,8 +405,9 @@ "metadata": {}, "outputs": [], "source": [ - "results_df.groupby(['Sex', 'Model'])[['Cohen Kappa', 'Accuracy', 'Macro F1', 'Balanced Accuracy']].agg(\n", - " lambda x: f\"{np.mean(x):.3f} \" + f\"± {np.std(x):.3f}\")" + "results_df.groupby([\"Sex\", \"Model\"])[\n", + " [\"Cohen Kappa\", \"Accuracy\", \"Macro F1\", \"Balanced Accuracy\"]\n", + "].agg(lambda x: f\"{np.mean(x):.3f} \" + f\"± {np.std(x):.3f}\")" ] }, { @@ -371,9 +423,26 @@ "metadata": {}, "outputs": [], "source": [ - "generate_confusion_matrices(results_df, ACTIVITY_LABELS, save_path=out_dir+\"/plots/conf_full_population.png\", fontsize=18)\n", - "generate_confusion_matrices(results_df, ACTIVITY_LABELS, group_by=\"Sex\", save_path=out_dir+\"/plots/conf_by_sex.png\", fontsize=18)\n", - "generate_confusion_matrices(results_df, ACTIVITY_LABELS, group_by=\"Age Band\", save_path=out_dir+\"/plots/conf_by_age.png\", fontsize=18)" + "generate_confusion_matrices(\n", + " results_df,\n", + " ACTIVITY_LABELS,\n", + " save_path=out_dir + \"/plots/conf_full_population.png\",\n", + " fontsize=18,\n", + ")\n", + "generate_confusion_matrices(\n", + " results_df,\n", + " ACTIVITY_LABELS,\n", + " group_by=\"Sex\",\n", + " save_path=out_dir + \"/plots/conf_by_sex.png\",\n", + " fontsize=18,\n", + ")\n", + "generate_confusion_matrices(\n", + " results_df,\n", + " ACTIVITY_LABELS,\n", + " group_by=\"Age Band\",\n", + " save_path=out_dir + \"/plots/conf_by_age.png\",\n", + " fontsize=18,\n", + ")" ] }, { @@ -389,7 +458,13 @@ "metadata": {}, "outputs": [], "source": [ - "generate_bland_altman_panel(results_df, ACTIVITY_LABELS, ANNOLABEL, save_path=out_dir+\"/plots/ba_panel_full_population.png\", fontsize=20)" + "generate_bland_altman_panel(\n", + " results_df,\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " save_path=out_dir + \"/plots/ba_panel_full_population.png\",\n", + " fontsize=20,\n", + ")" ] }, { @@ -398,11 +473,26 @@ "metadata": {}, "outputs": [], "source": [ - "generate_bland_altman_plots(results_df, ACTIVITY_LABELS, ANNOLABEL, save_path=out_dir+\"/plots/ba_full_population.png\")\n", - "generate_bland_altman_plots(results_df, ACTIVITY_LABELS, ANNOLABEL, \n", - " save_path=out_dir+\"/plots/ba_true_actinet_full_population.png\", compare_to_true='actinet')\n", - "generate_bland_altman_plots(results_df, ACTIVITY_LABELS, ANNOLABEL, \n", - " save_path=out_dir+\"/plots/ba_true_bbaa_full_population.png\", compare_to_true='bbaa')" + "generate_bland_altman_plots(\n", + " results_df,\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " save_path=out_dir + \"/plots/ba_full_population.png\",\n", + ")\n", + "generate_bland_altman_plots(\n", + " results_df,\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " save_path=out_dir + \"/plots/ba_true_actinet_full_population.png\",\n", + " compare_to_true=\"actinet\",\n", + ")\n", + "generate_bland_altman_plots(\n", + " results_df,\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " save_path=out_dir + \"/plots/ba_true_bbaa_full_population.png\",\n", + " compare_to_true=\"bbaa\",\n", + ")" ] }, { @@ -411,12 +501,27 @@ "metadata": {}, "outputs": [], "source": [ - "generate_bland_altman_plots(results_df[results_df[\"Sex\"]==\"Female\"], ACTIVITY_LABELS, ANNOLABEL, subset=\"female\",\n", - " save_path=out_dir+\"/plots/ba_by_sex_female.png\")\n", - "generate_bland_altman_plots(results_df[results_df[\"Sex\"]==\"Male\"], ACTIVITY_LABELS, ANNOLABEL, subset=\"male\",\n", - " save_path=out_dir+\"/plots/ba_by_sex_male.png\")\n", - "generate_bland_altman_plots(results_df, ACTIVITY_LABELS, ANNOLABEL, group_by=\"Age Band\", \n", - " save_path=out_dir+\"/plots/ba_by_age.png\")" + "generate_bland_altman_plots(\n", + " results_df[results_df[\"Sex\"] == \"Female\"],\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " subset=\"female\",\n", + " save_path=out_dir + \"/plots/ba_by_sex_female.png\",\n", + ")\n", + "generate_bland_altman_plots(\n", + " results_df[results_df[\"Sex\"] == \"Male\"],\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " subset=\"male\",\n", + " save_path=out_dir + \"/plots/ba_by_sex_male.png\",\n", + ")\n", + "generate_bland_altman_plots(\n", + " results_df,\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " group_by=\"Age Band\",\n", + " save_path=out_dir + \"/plots/ba_by_age.png\",\n", + ")" ] }, { @@ -425,12 +530,30 @@ "metadata": {}, "outputs": [], "source": [ - "generate_bland_altman_plots(results_df[results_df[\"Sex\"]==\"Female\"], ACTIVITY_LABELS, ANNOLABEL, subset=\"female\",\n", - " save_path=out_dir+\"/plots/ba_true_actinet_by_sex_female.png\", compare_to_true='actinet')\n", - "generate_bland_altman_plots(results_df[results_df[\"Sex\"]==\"Male\"], ACTIVITY_LABELS, ANNOLABEL, subset=\"male\",\n", - " save_path=out_dir+\"/plots/ba_true_actinet_by_sex_male.png\", compare_to_true='actinet')\n", - "generate_bland_altman_plots(results_df, ACTIVITY_LABELS, ANNOLABEL, group_by=\"Age Band\", \n", - " save_path=out_dir+\"/plots/ba_true_actinet_by_age.png\", compare_to_true='actinet')" + "generate_bland_altman_plots(\n", + " results_df[results_df[\"Sex\"] == \"Female\"],\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " subset=\"female\",\n", + " save_path=out_dir + \"/plots/ba_true_actinet_by_sex_female.png\",\n", + " compare_to_true=\"actinet\",\n", + ")\n", + "generate_bland_altman_plots(\n", + " results_df[results_df[\"Sex\"] == \"Male\"],\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " subset=\"male\",\n", + " save_path=out_dir + \"/plots/ba_true_actinet_by_sex_male.png\",\n", + " compare_to_true=\"actinet\",\n", + ")\n", + "generate_bland_altman_plots(\n", + " results_df,\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " group_by=\"Age Band\",\n", + " save_path=out_dir + \"/plots/ba_true_actinet_by_age.png\",\n", + " compare_to_true=\"actinet\",\n", + ")" ] }, { @@ -439,12 +562,30 @@ "metadata": {}, "outputs": [], "source": [ - "generate_bland_altman_plots(results_df[results_df[\"Sex\"]==\"Female\"], ACTIVITY_LABELS, ANNOLABEL, subset=\"female\",\n", - " save_path=out_dir+\"/plots/ba_true_bbaa_by_sex_female.png\", compare_to_true='bbaa')\n", - "generate_bland_altman_plots(results_df[results_df[\"Sex\"]==\"Male\"], ACTIVITY_LABELS, ANNOLABEL, subset=\"male\",\n", - " save_path=out_dir+\"/plots/ba_true_bbaa_by_sex_male.png\", compare_to_true='bbaa')\n", - "generate_bland_altman_plots(results_df, ACTIVITY_LABELS, ANNOLABEL, group_by=\"Age Band\", \n", - " save_path=out_dir+\"/plots/ba_true_bbaa_by_age.png\", compare_to_true='bbaa')" + "generate_bland_altman_plots(\n", + " results_df[results_df[\"Sex\"] == \"Female\"],\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " subset=\"female\",\n", + " save_path=out_dir + \"/plots/ba_true_bbaa_by_sex_female.png\",\n", + " compare_to_true=\"bbaa\",\n", + ")\n", + "generate_bland_altman_plots(\n", + " results_df[results_df[\"Sex\"] == \"Male\"],\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " subset=\"male\",\n", + " save_path=out_dir + \"/plots/ba_true_bbaa_by_sex_male.png\",\n", + " compare_to_true=\"bbaa\",\n", + ")\n", + "generate_bland_altman_plots(\n", + " results_df,\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " group_by=\"Age Band\",\n", + " save_path=out_dir + \"/plots/ba_true_bbaa_by_age.png\",\n", + " compare_to_true=\"bbaa\",\n", + ")" ] }, { @@ -469,7 +610,12 @@ "metadata": {}, "outputs": [], "source": [ - "plot_errors(results_df, ACTIVITY_LABELS, ANNOLABEL, save_path = out_dir + \"/plots/errors_full_population.png\")" + "plot_errors(\n", + " results_df,\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " save_path=out_dir + \"/plots/errors_full_population.png\",\n", + ")" ] }, { @@ -478,7 +624,13 @@ "metadata": {}, "outputs": [], "source": [ - "plot_errors(results_df, ACTIVITY_LABELS, ANNOLABEL, group_by='Age Band', save_path = out_dir + \"/plots/errors_by_age.png\")" + "plot_errors(\n", + " results_df,\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " group_by=\"Age Band\",\n", + " save_path=out_dir + \"/plots/errors_by_age.png\",\n", + ")" ] }, { @@ -487,7 +639,13 @@ "metadata": {}, "outputs": [], "source": [ - "plot_errors(results_df, ACTIVITY_LABELS, ANNOLABEL, group_by='Sex', save_path=out_dir + \"/plots/errors_by_sex.png\")" + "plot_errors(\n", + " results_df,\n", + " ACTIVITY_LABELS,\n", + " ANNOLABEL,\n", + " group_by=\"Sex\",\n", + " save_path=out_dir + \"/plots/errors_by_sex.png\",\n", + ")" ] } ], @@ -507,7 +665,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.9.25" } }, "nbformat": 4, diff --git a/src/actinet/utils/eval_utils.py b/src/actinet/utils/eval_utils.py index a89d502..9f661c6 100644 --- a/src/actinet/utils/eval_utils.py +++ b/src/actinet/utils/eval_utils.py @@ -11,7 +11,6 @@ balanced_accuracy_score, ) - Number = Union[int, float] @@ -40,8 +39,7 @@ def extract_activity_predictions( age_band: Any = None, sex: Any = None, return_true_labels: Literal[True] = True, -) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, int]: - ... +) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, int]: ... @overload @@ -51,8 +49,7 @@ def extract_activity_predictions( age_band: Any = None, sex: Any = None, return_true_labels: Literal[False] = False, -) -> Tuple[np.ndarray, np.ndarray, int]: - ... +) -> Tuple[np.ndarray, np.ndarray, int]: ... def extract_activity_predictions( @@ -120,13 +117,13 @@ def extract_activity_predictions( def build_mae_cell(true_values: Any, pred_values: Any) -> str: """Builds a MAE cell for a given set of true and predicted values.""" - mae = np.abs(true_values - pred_values) - return f"{np.mean(mae):.3f} ± {np.std(mae):.3f}" + ae = np.abs(true_values - pred_values) + return f"{np.mean(ae):.3f} ± {np.std(ae):.3f}" -def build_pvalue_cell(true_values: Any, pred_values: Any) -> str: - """Builds a p-value cell for a given set of true and predicted values.""" - _, p_value = stats.ttest_rel(true_values, pred_values) +def build_pvalue_cell(values1: Any, values2: Any) -> str: + """Builds a p-value cell for a given set of values.""" + _, p_value = stats.ttest_rel(values1, values2) if p_value < 0.001: return "<0.001" return f"{p_value:.3f}" @@ -134,7 +131,9 @@ def build_pvalue_cell(true_values: Any, pred_values: Any) -> str: def build_mae_table(df: pd.DataFrame, activities: Iterable[str]) -> pd.DataFrame: df_maes = pd.DataFrame( - columns=activities, index=["Baseline", "ActiNet", "p-value"], dtype=float + columns=activities, + index=["Baseline", "ActiNet", "p-value"], + dtype=float, ) for activity in activities: ( @@ -151,7 +150,8 @@ def build_mae_table(df: pd.DataFrame, activities: Iterable[str]) -> pd.DataFrame activity_actinet_true, activity_actinet_pred ) df_maes.loc["p-value", activity] = build_pvalue_cell( - activity_bbaa_true, activity_bbaa_pred + activity_actinet_true - activity_actinet_pred, + activity_bbaa_true - activity_bbaa_pred, ) return df_maes