project/02_3_grid_search_analysis.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "149ddbfc-c339-4843-8801-9eca57b0ff5e",
   "metadata": {},
   "source": [
    "# Analyis of grid hyperparameter search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f00a7b57-371d-4463-8da4-201e793f4401",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import pathlib\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import plotly.express as px\n",
    "import seaborn as sns\n",
    "import snakemake\n",
    "\n",
    "import pimmslearn.io\n",
    "import pimmslearn.nb\n",
    "import pimmslearn.pandas\n",
    "import pimmslearn.plotting.plotly as px_pimmslearn\n",
    "import pimmslearn.utils\n",
    "from pimmslearn import sampling\n",
    "from pimmslearn.analyzers import compare_predictions\n",
    "from pimmslearn.io import datasplits\n",
    "\n",
    "matplotlib.rcParams['figure.figsize'] = [12.0, 6.0]\n",
    "\n",
    "\n",
    "pd.options.display.max_columns = 45\n",
    "pd.options.display.max_rows = 100\n",
    "pd.options.display.multi_sparse = False\n",
    "\n",
    "logger = pimmslearn.logging.setup_nb_logger()\n",
    "logging.getLogger('fontTools').setLevel(logging.WARNING)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "4b5b7560-f825-4461-95ad-bf90accd2e87",
   "metadata": {},
   "source": [
    "## Papermill parameters"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "adfd7732-e5c1-45fb-aff3-4a5dc2446078",
   "metadata": {},
   "source": [
    "papermill parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6189a40-cfe2-4a53-9ce9-d7f86080bf79",
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "metrics_csv: str = \"path/to/all_metrics.csv\"  # file path to metrics\n",
    "configs_csv: str = \"path/to/all_configs.csv\"  # file path to configs (\"meta data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "333b0bba-e6ed-44bf-a7cc-f8f378c42d84",
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    assert pathlib.Path(metrics_csv).exists()\n",
    "    assert pathlib.Path(configs_csv).exists()\n",
    "except AssertionError:\n",
    "    metrics_csv = snakemake.input.metrics\n",
    "    configs_csv = snakemake.input.config\n",
    "    print(f\"{metrics_csv = }\", f\"{configs_csv = }\", sep=\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f0497b1-5f91-45e9-a3e1-88de08b928a9",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "# not robust\n",
    "try:\n",
    "    ORDER = {'model': snakemake.params.models}\n",
    "    FILE_FORMAT = snakemake.params.file_format\n",
    "except AttributeError:\n",
    "    ORDER = {'model': ['CF', 'DAE', 'VAE']}\n",
    "    FILE_FORMAT = 'csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db5c7467-ee5a-41e8-af87-192c7e9f82a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_metrics = pathlib.Path(metrics_csv)\n",
    "path_configs = pathlib.Path(configs_csv)\n",
    "FOLDER = path_metrics.parent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29aad0d1-c175-4b19-aa87-51140de3cf60",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "files_out = dict()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "4643bbea-09af-479a-b3e5-52fbd47fa6af",
   "metadata": {},
   "source": [
    "## Metrics of each run\n",
    "\n",
    "Metrics a `pandas.DataFrame`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52392b58",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics = pd.read_csv(path_metrics, index_col=0, header=[0, 1, 2])\n",
    "metrics.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bcc9a1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics.stack('model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d919f654",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ToDo: integrate as parameters\n",
    "metric_columns = ['MSE', 'MAE']\n",
    "model_keys = metrics.stack('model').index.levels[-1].unique().to_list()  # not used\n",
    "subset = metrics.columns.levels[0][0]\n",
    "print(f\"{subset = }\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "1c7278b3-ef8a-43f8-ade0-d8777b9b41b4",
   "metadata": {},
   "source": [
    "## Configuration of each run"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "c24e754e-9c0e-46dd-962e-084da27c9229",
   "metadata": {},
   "source": [
    "Experiment metadata from configs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "815da3a8-11e7-4ed4-bdaf-dc331eddc175",
   "metadata": {},
   "outputs": [],
   "source": [
    "meta = pd.read_csv(path_configs)\n",
    "meta['hidden_layers'] = (meta\n",
    "                         .loc[meta['hidden_layers'].notna(), 'hidden_layers']\n",
    "                         .apply(lambda x: tuple(eval(x)))\n",
    "                         )\n",
    "meta['n_hidden_layers'] = (meta\n",
    "                           .loc[meta['hidden_layers'].notna(), 'hidden_layers']\n",
    "                           .apply(len)\n",
    "                           )\n",
    "meta['n_hidden_layers'] = (meta\n",
    "                           ['n_hidden_layers']\n",
    "                           .fillna(0)\n",
    "                           .astype(int)\n",
    "                           )\n",
    "meta.loc[meta['hidden_layers'].isna(), 'hidden_layers'] = None\n",
    "meta = meta.set_index('id')\n",
    "meta"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "b4dd468f-8995-403d-a389-6c4e4e912cd5",
   "metadata": {},
   "source": [
    "Batch size for collab models depends on a factor (as the data in long\n",
    "format has roughly  N samples * M features entries)."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "e8c44238-09d1-4c79-95f3-597cce466cb2",
   "metadata": {},
   "source": [
    "## Colorcoded metrics\n",
    "\n",
    "- can be one of the [matplotlib color maps](https://matplotlib.org/stable/tutorials/colors/colormaps.html), which also have reversed version indicated by `*_r`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d256326-0f74-49f8-af2e-a2112b74cf23",
   "metadata": {},
   "outputs": [],
   "source": [
    "sel_meta = (meta\n",
    "            .loc[metrics.index, ['latent_dim', 'hidden_layers', 'batch_size']]\n",
    "            )\n",
    "sel_meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2a9c512-3d62-47bf-a2e6-cae812139181",
   "metadata": {},
   "outputs": [],
   "source": [
    "cmap = 'cividis_r'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fc40f89-177b-4750-8141-430ac49e5b3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ToDo: To make it cleaner: own config for each model (interpolated and median)\n",
    "metrics_styled = (metrics\n",
    "                  .set_index(\n",
    "                      pd.MultiIndex\n",
    "                      .from_frame(\n",
    "                          meta\n",
    "                          .loc[metrics.index, ['latent_dim', 'hidden_layers', 'batch_size']]\n",
    "                          # .loc[metrics.index]\n",
    "                      )\n",
    "                  )\n",
    "                  .sort_index()\n",
    "                  .stack('model')\n",
    "                  .drop_duplicates()\n",
    "                  .style.background_gradient(cmap)\n",
    "                  )\n",
    "\n",
    "metrics = metrics_styled.data\n",
    "metrics_styled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98751f1e-e3e3-42ba-87e2-0d1b33d71927",
   "metadata": {},
   "outputs": [],
   "source": [
    "fname = FOLDER / 'metrics_styled.xlsx'\n",
    "files_out['metrics_styled.xlsx'] = fname\n",
    "metrics_styled.to_excel(fname)\n",
    "logger.info(f\"Saved styled metrics: {fname}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "57933a40-4833-43f4-aa1c-79e20c64ed34",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Plot Top 10 for simulated Na validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ee241c3-ec2b-4a3c-bd7f-464dee83c750",
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = metrics[\"valid_fake_na\"].sort_values(\n",
    "    'MSE').iloc[:10, :-2].plot(rot=45,\n",
    "                               x_compat=False,\n",
    "                               xticks=list(range(10)),\n",
    "                               marker='o',\n",
    "                               linestyle='',\n",
    "                               )\n",
    "_ = ax.set_xticklabels(ax.get_xticklabels(), rotation=45,\n",
    "                       horizontalalignment='right')\n",
    "fig = ax.get_figure()\n",
    "fig.tight_layout()\n",
    "pimmslearn.savefig(fig, name='top_10_models_validation_fake_na', folder=FOLDER)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "16b21959-b63d-45ba-9c02-486c49f9e1f6",
   "metadata": {},
   "source": [
    "## Create metrics in long format\n",
    "\n",
    "To use colors meaningfully, the long format of the data is needed.\n",
    "\n",
    "Rebuild metrics from dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2279f91a-37fd-4fbf-875d-ed34a4c3d77d",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_long = pd.read_csv(path_metrics, index_col=[0], header=[0, 1, 2])\n",
    "# columns_names = ['subset', 'data_split', 'model', 'metric_name']\n",
    "columns_names = list(metrics_long.columns.names)\n",
    "metrics_long.sample(5) if len(metrics_long) > 15 else metrics_long"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "6862bcfa-c01b-43ed-8a51-ddb994533d5e",
   "metadata": {},
   "source": [
    "Combine with total number of simulated NAs the metric is based on (`N`) into single column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5077fed-91dd-4459-9e64-257114ef3efb",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_N = (metrics_long\n",
    "             .loc[:, pd.IndexSlice[:, :, 'N']]\n",
    "             .stack(['data_split', 'model'])\n",
    "             .reset_index()\n",
    "             .drop_duplicates()\n",
    "             .set_index(['id', 'data_split', 'model'])\n",
    "             .astype(int)\n",
    "             )\n",
    "metrics_N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "109e411b-7ab1-4d8e-ae62-4f72cc2911ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_prop = (metrics_long\n",
    "                .loc[:, pd.IndexSlice[:, :, 'prop']]\n",
    "                .stack(['data_split', 'model'])\n",
    "                .reset_index()\n",
    "                .drop_duplicates()\n",
    "                .set_index(['id', 'data_split', 'model'])\n",
    "                .astype(int)\n",
    "                )\n",
    "metrics_prop"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "1b761ec0-6f98-4cf7-a119-a9e3cfdca275",
   "metadata": {},
   "source": [
    "join total number of simulated NAs (`N`) used to compute metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8816c310-86ed-498b-9b3f-31ff2761317a",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_long = (metrics_long\n",
    "                .loc[:, pd.IndexSlice[:, :, metric_columns]]\n",
    "                .stack(metrics_long.columns.names)\n",
    "                .to_frame('metric_value')\n",
    "                .reset_index('metric_name')\n",
    "                .join(metrics_N)\n",
    "                )\n",
    "metrics_long"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "38ff17f3-2723-440c-9a26-1ca40ed5008b",
   "metadata": {},
   "source": [
    "join metadata for each metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46ccda5f-354e-40ae-8b5e-c37a34219a59",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_long = (metrics_long\n",
    "                .reset_index(['data_split'])\n",
    "                .join(meta.set_index('model', append=True))\n",
    "                ).reset_index('model')\n",
    "# metrics_long.index.name = 'id'\n",
    "metrics_long.sample(5)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "7b3a8b50-fa2e-4b37-bda5-464e337f6bbb",
   "metadata": {},
   "source": [
    "Combine number of parameters into one columns (they are mutually exclusive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc7fa778-b874-4348-85c9-9f5647fc3b28",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ToDo: Still hacky: every model needs a config file (add prop. imputed)\n",
    "# groupby 'id'\n",
    "cols = ['M', 'data', 'file_format', 'fn_rawfile_metadata',\n",
    "        'folder_data', 'folder_experiment',\n",
    "        'level', 'meta_cat_col', 'meta_date_col',\n",
    "        'out_figures', 'out_folder', 'out_metrics', 'out_models', 'out_preds',\n",
    "        'sample_idx_position', 'save_pred_real_na']\n",
    "metrics_long[cols] = metrics_long.groupby(level=0)[cols].fillna(method='pad')\n",
    "metrics_long.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2aeb2fa-e3bf-4244-8c66-683e843c1760",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ToDo: Ensure each model configuration saves a \"n_params\" argument\n",
    "mask = metrics_long.model == 'KNN'\n",
    "# at least overall (and 1 for the number of replicates?)\n",
    "distances?\n",
    "metrics_long.loc[mask, 'n_params'] = 1\n",
    "mask = metrics_long.model == 'Median'\n",
    "# number of features to calculate median of\n",
    "metrics_long.loc[mask, 'n_params'] = metrics_long.loc[mask, 'M']\n",
    "\n",
    "metrics_long[[*columns_names, 'n_params']]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "800e4833-67eb-4656-89fd-04fc602a4ad6",
   "metadata": {},
   "source": [
    "A a descriptive column describing the `subset` and the total number of simulated NAs in it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe64834f-b73f-47c2-b669-0c50b8aa6a07",
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_long['subset_w_N'] = 'N: ' + metrics_long['N'].apply(lambda x: f\"{x:,d}\")\n",
    "metrics_long[['subset_w_N']]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "8c09cd35-5dad-44e0-b0b1-f0e5f5f0c693",
   "metadata": {},
   "source": [
    "Save for later inspection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0513100-b917-4f3d-b85b-0e054c99f293",
   "metadata": {},
   "outputs": [],
   "source": [
    "fname = FOLDER / 'metrics_long_df.csv'\n",
    "files_out[fname.stem] = fname\n",
    "metrics_long.to_csv(fname)  # Should all the plots be done without the metrics?\n",
    "logger.info(f\"Saved metrics in long format: {fname}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "c4607c64-2e90-4ed6-b337-8e210d7c37de",
   "metadata": {},
   "source": [
    "# Collection of Performance plots\n",
    "\n",
    "- specify `labels_dict` for plotly plotting\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7bcf7405-e205-4558-9335-083b398b97c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_dict = {\"NA not interpolated valid_collab collab MSE\": 'MSE',\n",
    "               'batch_size': 'bs',\n",
    "               'n_hidden_layers': \"No. of hidden layers\",\n",
    "               'latent_dim': 'hidden layer dimension',\n",
    "               'subset_w_N': 'subset',\n",
    "               'n_params': 'no. of parameter',\n",
    "               \"metric_value\": 'value',\n",
    "               'metric_name': 'metric',\n",
    "               'freq': 'freq/feature prevalence (across samples)'}"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "b5ba3f23-458e-4b20-8d02-527ef24c03a2",
   "metadata": {},
   "source": [
    "## Plot hyperparameter search results - overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31231114-7184-4c38-a0c5-5d399a6ba901",
   "metadata": {},
   "outputs": [],
   "source": [
    "hover_data = {k: ':,d' for k in\n",
    "              ['hidden_layers',\n",
    "               'latent_dim', 'n_params',\n",
    "               'batch_size', 'N'\n",
    "               ]}\n",
    "hover_data['data_split'] = True\n",
    "hover_data['metric_value'] = ':.4f'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8190d51-c4db-4aae-8b91-11641958a0f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "view = metrics_long[[\"model\", \"n_params\", \"data_split\", \"metric_name\", \"metric_value\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f98b49d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams['figure.figsize'] = (7, 4)\n",
    "plt.rcParams['lines.linewidth'] = 2\n",
    "plt.rcParams['lines.markersize'] = 3\n",
    "pimmslearn.plotting.make_large_descriptors(7)\n",
    "\n",
    "col_order = ('valid_fake_na', 'test_fake_na')\n",
    "row_order = ('MAE', 'MSE')\n",
    "fg = sns.relplot(\n",
    "    data=view,\n",
    "    x='n_params',\n",
    "    y='metric_value',\n",
    "    col=\"data_split\",\n",
    "    col_order=col_order,\n",
    "    row=\"metric_name\",\n",
    "    row_order=row_order,\n",
    "    hue=\"model\",\n",
    "    # style=\"day\",\n",
    "    palette=pimmslearn.plotting.defaults.color_model_mapping,\n",
    "    height=2,\n",
    "    aspect=1.8,\n",
    "    kind=\"scatter\",\n",
    ")\n",
    "\n",
    "(ax_00, ax_01), (ax_10, ax_11) = fg.axes\n",
    "ax_00.set_ylabel(row_order[0])\n",
    "ax_10.set_ylabel(row_order[1])\n",
    "_ = ax_00.set_title('validation data')  # col_order[0]\n",
    "_ = ax_01.set_title('test data')  # col_order[1]\n",
    "ax_10.set_xticklabels(ax_10.get_xticklabels(),\n",
    "                      rotation=45,\n",
    "                      horizontalalignment='right')\n",
    "ax_10.set_xlabel('number of parameters')  # n_params\n",
    "ax_11.set_xticklabels(ax_11.get_xticklabels(),\n",
    "                      rotation=45,\n",
    "                      horizontalalignment='right')\n",
    "ax_11.set_xlabel('number of parameters')\n",
    "ax_10.xaxis.set_major_formatter(\"{x:,.0f}\")\n",
    "ax_11.xaxis.set_major_formatter(\"{x:,.0f}\")\n",
    "_ = ax_10.set_title('')\n",
    "_ = ax_11.set_title('')\n",
    "fg.tight_layout()\n",
    "fname\n",
    "fname = FOLDER / \"hyperpar_results_by_parameters_val+test.pdf\"\n",
    "files_out[fname.name] = fname.as_posix()\n",
    "view.to_excel(fname.with_suffix('.xlsx'))\n",
    "fg.savefig(fname)\n",
    "fg.savefig(fname.with_suffix('.png'), dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa98bb4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def plot_by_params(data_split: str = '', subset: str = ''):\n",
    "    selected = metrics_long\n",
    "    if data_split:\n",
    "        selected = selected.query(f'data_split == \"{data_split}\"')\n",
    "    if subset:\n",
    "        selected = selected.query(f'subset == \"{subset}\"')\n",
    "    fig = px.scatter(selected,\n",
    "                     x='n_params',\n",
    "                     y='metric_value',\n",
    "                     color=\"model\",\n",
    "                     facet_row=\"metric_name\",\n",
    "                     # facet_col=\"subset_w_N\", \"N\", \"prop\"\n",
    "                     hover_data=hover_data,\n",
    "                     title=f'Performance by number of parameters for {data_split.replace(\"_\", \" \")} data'.replace(\n",
    "                         \"  \", \" \"),\n",
    "                     labels=labels_dict,\n",
    "                     category_orders=ORDER,\n",
    "                     width=750,\n",
    "                     height=300,\n",
    "                     template='none',\n",
    "                     )\n",
    "    fig.update_traces(marker={'size': 3})\n",
    "    fig.update_layout(\n",
    "        font={'size': 8},\n",
    "        xaxis={'title': {'standoff': 6}},\n",
    "        yaxis={'title': {'standoff': 6}})\n",
    "    return fig\n",
    "\n",
    "\n",
    "dataset = \"test_fake_na\"\n",
    "fig = plot_by_params(dataset)\n",
    "fname = FOLDER / f\"hyperpar_{dataset}_results_by_parameters.pdf\"\n",
    "files_out[f\"hyperpar_{dataset}_results_by_parameters.pdf\"] = fname\n",
    "fig.write_image(fname)\n",
    "logger.info(f\"Save to {fname}\")\n",
    "fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d309a52f-f955-4ff9-ae1b-e6082b451dd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = \"valid_fake_na\"\n",
    "fig = plot_by_params(dataset)\n",
    "fname = (FOLDER /\n",
    "         f\"hyperpar_{dataset}_results_by_parameters.pdf\")\n",
    "files_out[f\"hyperpar_{dataset}_results_by_parameters.pdf\"] = fname\n",
    "fig.write_image(fname)\n",
    "logger.info(f\"Save to {fname}\")\n",
    "fig"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "47534519",
   "metadata": {},
   "source": [
    "## Select best model for each `latent_dim`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4bdcc559-28de-4056-84b3-5f24ea175fa9",
   "metadata": {},
   "outputs": [],
   "source": [
    "group_by = ['data_split', 'latent_dim', 'metric_name', 'model']\n",
    "metrics_long_sel_min = metrics_long.reset_index(\n",
    ").groupby(by=group_by\n",
    "          ).apply(lambda df: df.sort_values(by='metric_value').iloc[0])\n",
    "metrics_long_sel_min"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e43eceaa-ee22-456c-a3ca-1e830c17f750",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_plotly_figure(dataset: str, x='latent_dim'):\n",
    "    fig = px.scatter(metrics_long_sel_min.loc[dataset],\n",
    "                     x=x,\n",
    "                     y='metric_value',\n",
    "                     color=\"model\",\n",
    "                     facet_row=\"metric_name\",\n",
    "                     # facet_col=\"subset_w_N\",\n",
    "                     hover_data=hover_data,\n",
    "                     title=f'Performance on {dataset.replace(\"_\", \" \")} data',\n",
    "                     labels=labels_dict,\n",
    "                     category_orders=ORDER,\n",
    "                     width=750,\n",
    "                     height=300,\n",
    "                     template='none',\n",
    "                     )\n",
    "    fig.update_xaxes(dict(\n",
    "        tickmode='array',\n",
    "        tickvals=sorted(metrics_long[x].unique()),\n",
    "    )\n",
    "    )\n",
    "    fig.update_traces(marker={'size': 3})\n",
    "    fig.update_layout(\n",
    "        font={'size': 8},\n",
    "        xaxis={'title': {'standoff': 6}},\n",
    "        yaxis={'title': {'standoff': 6}})\n",
    "    return fig\n",
    "\n",
    "\n",
    "dataset = 'test_fake_na'\n",
    "fig = get_plotly_figure(dataset)\n",
    "fname = FOLDER / f\"hyperpar_{dataset}_results_best.pdf\"\n",
    "files_out[f\"hyperpar_{dataset}_results_best.pdf\"] = fname\n",
    "fig.write_image(fname)\n",
    "logger.info(f\"Save to {fname}\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7061609a-7179-45dc-a8eb-3172731fce1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'valid_fake_na'\n",
    "fig = get_plotly_figure(dataset)\n",
    "fname = FOLDER / f\"hyperpar_{dataset}_results_best.pdf\"\n",
    "files_out[f\"hyperpar_{dataset}_results_best.pdf\"] = fname\n",
    "fig.write_image(fname)\n",
    "logger.info(f\"Save to {fname}\")\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "afa690af-5dcc-4857-a248-1860999f79d1",
   "metadata": {},
   "source": [
    "## Performance along feature prevalence in training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "655092b2-0ba2-4ff0-b408-d982df1501c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'valid_fake_na'\n",
    "group_by = ['data_split', 'metric_name', 'model', 'latent_dim']\n",
    "METRIC = 'MAE'  # params.metric\n",
    "selected = (metrics_long\n",
    "            .reset_index()\n",
    "            .groupby(by=group_by)\n",
    "            .apply(lambda df: df.sort_values(by='metric_value').iloc[0])\n",
    "            .loc[dataset])\n",
    "fname = FOLDER / 'best_models_metrics_per_latent.csv'\n",
    "files_out['best_models_metrics_per_latent.csv'] = fname\n",
    "selected.to_csv(fname)\n",
    "selected.sample(5) if len(selected) > 5 else selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "208805b3-79f1-4b45-8653-fa4acf5b0e4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_with_latent = list(selected['model'].unique())\n",
    "model_with_latent"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "bf60875d-7062-4a51-abdf-59d3f8463b67",
   "metadata": {},
   "source": [
    "### For best latent dimension (on average)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "68b26f17-7912-49f3-a2cd-07c8115bfa00",
   "metadata": {},
   "source": [
    "select minimum value of latent dim over trained models on average\n",
    " 1. select for each latent the best model configuration (by DL model)\n",
    " 2. Choose the on average best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d98e3210-bafe-4456-b7e8-2bad861d0d71",
   "metadata": {},
   "outputs": [],
   "source": [
    "min_latent = (selected\n",
    "              .loc[METRIC]\n",
    "              .loc[model_with_latent]\n",
    "              .groupby(level='latent_dim')\n",
    "              .agg({'metric_value': 'mean'})\n",
    "              .sort_values('metric_value')\n",
    "              )\n",
    "min_latent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61abef54-3dff-4b89-b4df-50bc6ceb87a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "min_latent = min_latent.index[0]\n",
    "print(\"Minimum latent value for average of models:\", min_latent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c19d4f94-2734-4352-a71e-4574aa1742ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected = (selected\n",
    "            .loc['MAE']\n",
    "            .loc[model_with_latent]\n",
    "            .loc[pd.IndexSlice[:, min_latent], :]\n",
    "            )\n",
    "selected"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "58717630-1cd3-49c4-bbb1-6fd72c3688b1",
   "metadata": {},
   "source": [
    "load predictions (could be made better)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc4f0b38-c04f-4161-b8f9-b9c224b45bfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'test_fake_na'  # load test split predictions\n",
    "selected['pred_to_load'] = (\n",
    "    selected['out_preds']\n",
    "    + ('/pred_val_' if 'val' in dataset else '/pred_test_')  # not good...\n",
    "    # + selected['hidden_layers'].apply(lambda s: '_hl_' + '_'.join(str(x)\n",
    "    #                                   for x in s) + '_' if s is not None else '_')\n",
    "    + selected.model\n",
    "    + '.csv'\n",
    ")\n",
    "selected['pred_to_load'].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4dd2b861-4c89-4477-8961-b5009afaacc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "248050bc-5dfc-4923-aa16-7bc4888813d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper = {k: f'{k} - ' + \"HL: {}\".format(\n",
    "    str(selected.loc[k, 'hidden_layers'].to_list()[0]))\n",
    "    for k in selected.model\n",
    "}\n",
    "mapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82f6ccf2-7e77-4592-bb9a-fcedb35143fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "order = ['observed'] + [m for m in ORDER['model'] if m in selected['model']]\n",
    "order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0daea0ae-755d-4787-a319-4c737edc1252",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_split = compare_predictions.load_predictions(\n",
    "    selected['pred_to_load'].to_list())[[*order]]\n",
    "pred_split = pred_split.rename(mapper, axis=1)\n",
    "order = list(pred_split.columns[1:])\n",
    "pred_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7daaae4-0519-4346-a025-a8647545eb72",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = datasplits.DataSplits.from_folder(\n",
    "    FOLDER / 'data',\n",
    "    # ! fileformat can be different\n",
    "    file_format=FILE_FORMAT)\n",
    "\n",
    "N_SAMPLES = int(data.train_X.index.levels[0].nunique())\n",
    "# selection criteria # maybe to be set externally (depends on data selection)\n",
    "FREQ_MIN = int(N_SAMPLES * 0.25)\n",
    "\n",
    "logger.info(\n",
    "    f\"N Samples: {N_SAMPLES:,d} - set minumum: {FREQ_MIN:,d} for plotting.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5a09507-9cd0-4210-ba10-8cd514a9ed1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "freq_feat = sampling.frequency_by_index(data.train_X, 0)\n",
    "freq_feat.name = 'freq'\n",
    "# freq_feat = pimmslearn.io.datasplits.load_freq(data_folder) # could be loaded from datafolder\n",
    "freq_feat.head()  # training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea944a52-9a1c-4375-9ca8-f759edf6933b",
   "metadata": {},
   "outputs": [],
   "source": [
    "errors = pimmslearn.pandas.calc_errors_per_feat(\n",
    "    pred=pred_split, freq_feat=freq_feat, target_col='observed')\n",
    "errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "815abad4-4224-4c7c-a793-dc949ad73940",
   "metadata": {},
   "outputs": [],
   "source": [
    "files_out[f'n_obs_error_counts_{dataset}.pdf'] = (FOLDER /\n",
    "                                                  f'n_obs_error_counts_{dataset}.pdf')\n",
    "ax = (errors['n_obs']\n",
    "      .value_counts()\n",
    "      .sort_index()\n",
    "      .plot(style='.',\n",
    "            xlabel='number of samples',\n",
    "            ylabel='observations')\n",
    "      )\n",
    "pimmslearn.savefig(ax.get_figure(), files_out[f'n_obs_error_counts_{dataset}.pdf'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47911dc8-f434-4d45-963d-579b75444508",
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = errors.plot.scatter('freq', 'n_obs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e803cf3-93ca-4588-9eed-0f43cf1ab593",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_obs_error_is_based_on = errors['n_obs']\n",
    "errors = errors.drop('n_obs', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52170533-1a9d-469a-9300-8811f50fc628",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "M_feat = len(errors)\n",
    "window_size = int(M_feat / 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95e65cb0-d3c1-4c60-b34f-9507de6becb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "errors_smoothed = errors.copy()\n",
    "# errors_smoothed[errors.columns[:-1]] = errors[errors.columns[:-1]].rolling(window=window_size, min_periods=1).mean()\n",
    "errors_smoothed[order] = errors[order].rolling(\n",
    "    window=window_size, min_periods=1).mean()\n",
    "errors_smoothed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7743f5b0-ce83-4471-a69b-4e9f55f49467",
   "metadata": {},
   "outputs": [],
   "source": [
    "errors_smoothed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22bfa6d1-503a-412c-a01f-8739e318b6c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = errors_smoothed[freq_feat.name] >= FREQ_MIN\n",
    "ax = (errors_smoothed\n",
    "      .loc[mask]\n",
    "      .rename_axis('', axis=1)\n",
    "      .plot(x=freq_feat.name,\n",
    "            xlabel='freq/feature prevalence (across samples)',\n",
    "            ylabel=f'rolling average error ({METRIC})',\n",
    "            xlim=(\n",
    "                FREQ_MIN, errors_smoothed[freq_feat.name].max()),\n",
    "            # title=f'Rolling average error by feature frequency {msg_annotation}'\n",
    "            ))\n",
    "\n",
    "msg_annotation = f\"(Latend dim: {min_latent}, No. of feat: {M_feat}, window_size: {window_size})\"\n",
    "print(msg_annotation)\n",
    "\n",
    "files_out[f'best_models_ld_{min_latent}_rolling_errors_by_freq'] = (\n",
    "    FOLDER / f'best_models_ld_{min_latent}_rolling_errors_by_freq')\n",
    "pimmslearn.savefig(\n",
    "    ax.get_figure(),\n",
    "    name=files_out[f'best_models_ld_{min_latent}_rolling_errors_by_freq'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d38a4be1-26d2-4983-ac39-b32d4918e2fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "errors_smoothed_long = errors_smoothed.drop('freq', axis=1).stack().to_frame(\n",
    "    'rolling error average').reset_index(-1).join(freq_feat)\n",
    "errors_smoothed_long"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "8d646009-eff7-46f2-b3f4-7f28782e2435",
   "metadata": {},
   "source": [
    "Save html versin of curve with annotation of errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2199ed3-7593-49f8-8620-f8eb779250fa",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig = px.line((errors_smoothed_long\n",
    "               .loc[errors_smoothed_long[freq_feat.name] >= FREQ_MIN]\n",
    "               .join(n_obs_error_is_based_on)\n",
    "               .sort_values(by='freq')),\n",
    "              x=freq_feat.name,\n",
    "              color='model',\n",
    "              y='rolling error average',\n",
    "              hover_data=['n_obs'],\n",
    "              # title=f'Rolling average error by feature frequency {msg_annotation}',\n",
    "              labels=labels_dict,\n",
    "              category_orders={'model': order},\n",
    "              )\n",
    "fig = px_pimmslearn.apply_default_layout(fig)\n",
    "fig.update_layout(legend_title_text='')  # remove legend title\n",
    "files_out[f'best_models_ld_{min_latent}_errors_by_freq_plotly.html'] = (\n",
    "    FOLDER / f'best_models_ld_{min_latent}_errors_by_freq_plotly.html')\n",
    "fig.write_html(\n",
    "    files_out[f'best_models_ld_{min_latent}_errors_by_freq_plotly.html'])\n",
    "fig"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "97748fea",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "source": [
    "#### Average error by feature frequency.\n",
    "Group all features with same frequency and calculate average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f68be9bc-4327-4df9-bc6b-67d39fd77318",
   "metadata": {},
   "outputs": [],
   "source": [
    "errors_smoothed = errors.copy()\n",
    "\n",
    "ax = errors_smoothed.loc[errors_smoothed['freq'] >= FREQ_MIN].groupby(by='freq'\n",
    "                                                                      ).mean(\n",
    ").sort_index(\n",
    ").rolling(window=3, min_periods=1\n",
    "          ).mean(\n",
    ").rename_axis('', axis=1\n",
    "              ).plot(\n",
    "    xlabel='freq/ feature prevalence (across samples)',\n",
    "    ylabel='rolling error average',\n",
    "    # title='mean error for features averaged for each frequency'\n",
    "    xlim=(FREQ_MIN, freq_feat.max())\n",
    ")\n",
    "files_out[f'best_models_ld_{min_latent}_errors_by_freq_averaged'] = (\n",
    "    FOLDER / f'best_models_ld_{min_latent}_errors_by_freq_averaged')\n",
    "pimmslearn.savefig(\n",
    "    ax.get_figure(),\n",
    "    files_out[f'best_models_ld_{min_latent}_errors_by_freq_averaged'])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "cfb3550a-059a-4bbd-a019-2a4423da1b99",
   "metadata": {},
   "source": [
    "### For best models per model class\n",
    "\n",
    "- select on validation data, report on prediction on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d18f487c-495e-42d7-8a8f-2fb6d910c857",
   "metadata": {},
   "outputs": [],
   "source": [
    "group_by = ['data_split', 'metric_name', 'model']\n",
    "dataset = 'valid_fake_na'  # select on validation split\n",
    "\n",
    "\n",
    "selected = metrics_long.reset_index(\n",
    ").groupby(by=group_by\n",
    "          ).apply(lambda df: df.sort_values(by='metric_value').iloc[0]).loc[dataset]\n",
    "\n",
    "selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9f4fc8b-5d12-41ee-a2c2-8e4492ab5fc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "order_categories = {'data level': ['proteinGroups', 'peptides', 'evidence'],\n",
    "                    'model': ORDER['model']}\n",
    "order_models = set(selected['model'])\n",
    "order_models = [m for m in ORDER['model'] if m in order_models]\n",
    "\n",
    "selected = selected.loc['MAE'].loc[order_models]\n",
    "selected.to_csv(FOLDER / 'best_models_metrics.csv')\n",
    "selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e10c4d2-fb7a-402b-ad9c-3f8cf03a3896",
   "metadata": {},
   "outputs": [],
   "source": [
    "order_models = [m for m in order_models if m in selected['model']]\n",
    "selected = selected.loc[order_models]\n",
    "selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c64c09c6-acd3-46c1-b9d1-c26cb0c4b152",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'test_fake_na'  # load test split predictions\n",
    "selected['pred_to_load'] = (\n",
    "    selected['out_preds']\n",
    "    + ('/pred_val_' if 'val' in dataset else '/pred_test_')  # not good...\"\n",
    "    # + selected['hidden_layers'].apply(lambda s: '_hl_' + '_'.join(str(x)\n",
    "    #                                   for x in s) + '_' if s is not None else '_')\n",
    "    + selected.model\n",
    "    + '.csv'\n",
    ")\n",
    "selected['pred_to_load'].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a634329e-27af-4c1c-86e3-5a8145d2fdcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "sel_pred_to_load = []\n",
    "\n",
    "for fname in selected['pred_to_load']:\n",
    "    fname = pathlib.Path(fname)\n",
    "    if fname.exists():\n",
    "        sel_pred_to_load.append(fname.as_posix())\n",
    "    else:\n",
    "        logger.warning(f\"Missing prediction file: {fname}\")\n",
    "sel_pred_to_load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "848c8206-37fe-401c-b9d4-d450388571cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper = {k: f'{k} - LD: {selected.loc[k, \"latent_dim\"]} - HL: {selected.loc[k, \"hidden_layers\"]} '\n",
    "          for k in selected.model\n",
    "          }\n",
    "mapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6831af84-5ace-4191-9b8d-eb5f518a8d7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_split = compare_predictions.load_predictions(\n",
    "    sel_pred_to_load, shared_columns=['observed'])[['observed', *order_models]]\n",
    "pred_split = pred_split.rename(mapper, axis=1)\n",
    "order_models = list(pred_split.columns[1:])\n",
    "pred_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3d0299b-6188-451d-9a52-5e6e188354a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_count = pred_split.groupby(\n",
    "    by=pred_split.index.names[-1])[pred_split.columns[0]].count()\n",
    "ax = feat_count.hist(legend=False)\n",
    "ax.set_xlabel('feat used for comparison (in split)')\n",
    "ax.set_ylabel('observations')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31c1bcf4-5022-4dfb-89ba-f5279d8a8a14",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loaded above\n",
    "freq_feat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e69004a4-d9c6-4c4e-9508-9bb30ff4364b",
   "metadata": {},
   "outputs": [],
   "source": [
    "errors = pimmslearn.pandas.calc_errors_per_feat(\n",
    "    pred=pred_split, freq_feat=freq_feat, target_col='observed')\n",
    "idx_name = errors.index.name\n",
    "errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "497ea5d5-1b4f-48fa-8d71-4c52c0a8cacf",
   "metadata": {},
   "outputs": [],
   "source": [
    "files_out[f'best_models_errors_counts_obs_{dataset}.pdf'] = (FOLDER /\n",
    "                                                             f'n_obs_error_counts_{dataset}.pdf')\n",
    "ax = errors['n_obs'].value_counts().sort_index().plot(style='.')\n",
    "pimmslearn.savefig(ax.get_figure(),\n",
    "                   files_out[f'best_models_errors_counts_obs_{dataset}.pdf'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70524224-eadf-4c76-aec2-a2ea0e92360f",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_obs_error_is_based_on = errors['n_obs']\n",
    "errors = errors.drop('n_obs', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "427806b6-f8c9-44d5-9641-1cb742f254e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# shoudl be the same\n",
    "M_feat = len(errors)\n",
    "window_size = int(M_feat / 50)\n",
    "print(\n",
    "    f\"Features in split: {M_feat}, set window size for smoothing: {window_size}\")\n",
    "msg_annotation = f\"(No. of feat: {M_feat}, window_size: {window_size})\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df01642a-726f-4375-8057-749a8f046d18",
   "metadata": {},
   "outputs": [],
   "source": [
    "errors_smoothed = errors.copy()\n",
    "errors_smoothed[order_models] = errors[order_models].rolling(\n",
    "    window=window_size, min_periods=1).mean()\n",
    "mask = errors_smoothed[freq_feat.name] >= FREQ_MIN\n",
    "ax = (errors_smoothed\n",
    "      .loc[mask]\n",
    "      .rename_axis('', axis=1)\n",
    "      .plot(x=freq_feat.name,\n",
    "            ylabel='rolling error average',\n",
    "            xlabel='freq/feature prevalence (across samples)',\n",
    "            xlim=(\n",
    "                FREQ_MIN, freq_feat.max()),\n",
    "            # title=f'Rolling average error by feature frequency {msg_annotation}'\n",
    "            ))\n",
    "\n",
    "pimmslearn.savefig(\n",
    "    ax.get_figure(),\n",
    "    folder=FOLDER,\n",
    "    name=f'best_models_rolling_errors_{dataset}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eea784fe-018a-4baa-95c0-75125ac1de17",
   "metadata": {},
   "outputs": [],
   "source": [
    "errors_smoothed_long = errors_smoothed.drop('freq', axis=1).stack().to_frame(\n",
    "    'rolling error average').reset_index(-1).join(freq_feat).join(feat_count).reset_index()\n",
    "errors_smoothed_long"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "4321ff5e-f9ad-48f4-8eb1-d78d6a01daf5",
   "metadata": {},
   "source": [
    "Save html versin of curve with annotation of errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc280dd7-a018-430f-bf71-bdb0e5ed2794",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.line((errors_smoothed_long.loc[errors_smoothed_long[freq_feat.name] >= FREQ_MIN]\n",
    "               .join(n_obs_error_is_based_on)\n",
    "               .sort_values(by='freq')),\n",
    "              x=freq_feat.name,\n",
    "              color='model',\n",
    "              y='rolling error average',\n",
    "              title=f'Rolling average error by feature frequency {msg_annotation}',\n",
    "              labels=labels_dict,\n",
    "              hover_data=[feat_count.name, idx_name, 'n_obs'],\n",
    "              category_orders={'model': order_models})\n",
    "fig = px.apply_default_layout(fig)\n",
    "fig.update_layout(legend_title_text='')  # remove legend title\n",
    "files_out[f'best_models_errors_{dataset}_by_freq_plotly.html'] = (FOLDER /\n",
    "                                                                  f'best_models_errors_{dataset}_by_freq_plotly.html')\n",
    "# fig.write_image(FOLDER / f'best_models_errors_{dataset}_by_freq_plotly.pdf')\n",
    "fig.write_html(files_out[f'best_models_errors_{dataset}_by_freq_plotly.html'])\n",
    "fig"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "00a3c083-cdad-4ff3-a1d1-d7f9110f59bd",
   "metadata": {},
   "source": [
    "## Correlation plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "877712f6-a4a1-428b-b944-748cf7122704",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_split"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "ec8f6ba9-85d8-41a7-b909-1fee89ee3344",
   "metadata": {},
   "source": [
    "### by feature across samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4af7b37-456d-45cc-abaf-68a0275d10b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_per_feat = pred_split.groupby(idx_name).aggregate(\n",
    "    lambda df: df.corr().loc['observed'])[order_models]\n",
    "corr_per_feat = corr_per_feat.join(pred_split.groupby(idx_name)[\n",
    "                                   'observed'].count().rename('n_obs'))\n",
    "too_few_obs = corr_per_feat['n_obs'] < 3\n",
    "corr_per_feat.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60519011-1aba-4e6c-8495-3ccb6454e307",
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_per_feat.loc[~too_few_obs].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "195d9117-c1dc-49ef-8acb-6f76d1e56970",
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_per_feat.loc[too_few_obs].dropna(thresh=3, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aeeb84ed-33fe-4b7f-b8f6-bbe6e2687bd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "figsize = 8, 8  # None\n",
    "fig, ax = plt.subplots(figsize=figsize)\n",
    "\n",
    "kwargs = dict(rot=45,\n",
    "              # title='Corr. betw. simulated NA and model pred. per feat',\n",
    "              ylabel=f'correlation per feature ({idx_name})')\n",
    "ax = corr_per_feat.loc[~too_few_obs].drop(\n",
    "    'n_obs', axis=1).plot.box(**kwargs, ax=ax)\n",
    "_ = ax.set_xticklabels(ax.get_xticklabels(), rotation=45,\n",
    "                       horizontalalignment='right')\n",
    "files_out[f'pred_corr_per_feat_{dataset}'] = (FOLDER /\n",
    "                                              f'pred_corr_per_feat_{dataset}')\n",
    "pimmslearn.savefig(ax.get_figure(), name=files_out[f'pred_corr_per_feat_{dataset}'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51415a04-7840-4132-945e-cd1de04de7b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "files_out[f'pred_corr_per_feat_{dataset}.xlsx'] = (FOLDER /\n",
    "                                                   f'pred_corr_per_feat_{dataset}.xlsx')\n",
    "with pd.ExcelWriter(files_out[f'pred_corr_per_feat_{dataset}.xlsx']) as writer:\n",
    "    corr_per_feat.loc[~too_few_obs].describe().to_excel(\n",
    "        writer, sheet_name='summary')  # excluded -1 and 1 version\n",
    "    # complete information\n",
    "    corr_per_feat.to_excel(writer, sheet_name='correlations')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "b375fcbc-6cf0-4797-be5c-0ddd25468f75",
   "metadata": {},
   "source": [
    "### within sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8e1de68-71ba-4868-99c0-54201e8a20af",
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_per_sample = pred_split.groupby('Sample ID').aggregate(\n",
    "    lambda df: df.corr().loc['observed'])[order_models]\n",
    "corr_per_sample = corr_per_sample.join(pred_split.groupby('Sample ID')[\n",
    "                                       'observed'].count().rename('n_obs'))\n",
    "corr_per_sample.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "803aea2b-1a56-4fc7-ac7a-c504949f3e52",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=figsize)\n",
    "\n",
    "kwargs = dict(ylim=(0.7, 1), rot=45,\n",
    "              # title='Corr. betw. simulated NA and model pred. per sample',\n",
    "              ylabel='correlation per sample')\n",
    "ax = corr_per_sample.drop('n_obs', axis=1).plot.box(**kwargs, ax=ax)\n",
    "_ = ax.set_xticklabels(ax.get_xticklabels(), rotation=45,\n",
    "                       horizontalalignment='right')\n",
    "files_out[f'pred_corr_per_sample_{dataset}'] = (FOLDER /\n",
    "                                                f'pred_corr_per_sample_{dataset}')\n",
    "pimmslearn.savefig(ax.get_figure(),\n",
    "                   name=files_out[f'pred_corr_per_sample_{dataset}'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3ad6407-2eb0-4ad2-bd43-d6938b6139d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "files_out[f'pred_corr_per_sample_{dataset}.xlsx'] = (FOLDER /\n",
    "                                                     f'pred_corr_per_sample_{dataset}.xlsx')\n",
    "with pd.ExcelWriter(files_out[f'pred_corr_per_sample_{dataset}.xlsx']) as writer:\n",
    "    corr_per_sample.describe().to_excel(writer, sheet_name='summary')\n",
    "    corr_per_sample.to_excel(writer, sheet_name='correlations')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "432b7824-12c0-41a5-87a7-9d921ee8e5f9",
   "metadata": {},
   "source": [
    "# Files written to disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47abbd4b-c617-4be1-8b5a-d1f1963d9607",
   "metadata": {},
   "outputs": [],
   "source": [
    "files_out"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "formats": "ipynb,py:percent"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "vscode": {
   "interpreter": {
    "hash": "cf83e9cb890c7f96eb0ae04f39a82254555f56a1a0ed2f03b23a8b40fe6cd31c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}