io.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "V28"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5-oBh3gKFKIz",
        "outputId": "d16ca6ce-b321-406c-bedd-706bc09dc62a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (2.2.2)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.11/dist-packages (3.10.0)\n",
            "Requirement already satisfied: seaborn in /usr/local/lib/python3.11/dist-packages (0.13.2)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.11/dist-packages (3.4.2)\n",
            "Collecting shap\n",
            "  Downloading shap-0.46.0-cp311-cp311-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (24 kB)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.11/dist-packages (1.6.1)\n",
            "Requirement already satisfied: plotly in /usr/local/lib/python3.11/dist-packages (5.24.1)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.11/dist-packages (1.13.1)\n",
            "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (1.26.4)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (2.9.0.post0)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas) (2025.1)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas) (2025.1)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.3.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (4.56.0)\n",
            "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.4.8)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (24.2)\n",
            "Requirement already satisfied: pillow>=8 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (11.1.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (3.2.1)\n",
            "Requirement already satisfied: tqdm>=4.27.0 in /usr/local/lib/python3.11/dist-packages (from shap) (4.67.1)\n",
            "Collecting slicer==0.0.8 (from shap)\n",
            "  Downloading slicer-0.0.8-py3-none-any.whl.metadata (4.0 kB)\n",
            "Requirement already satisfied: numba in /usr/local/lib/python3.11/dist-packages (from shap) (0.61.0)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.11/dist-packages (from shap) (3.1.1)\n",
            "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (1.4.2)\n",
            "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (3.5.0)\n",
            "Requirement already satisfied: tenacity>=6.2.0 in /usr/local/lib/python3.11/dist-packages (from plotly) (9.0.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/dist-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n",
            "Requirement already satisfied: llvmlite<0.45,>=0.44.0dev0 in /usr/local/lib/python3.11/dist-packages (from numba->shap) (0.44.0)\n",
            "Downloading shap-0.46.0-cp311-cp311-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (540 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m540.2/540.2 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading slicer-0.0.8-py3-none-any.whl (15 kB)\n",
            "Installing collected packages: slicer, shap\n",
            "Successfully installed shap-0.46.0 slicer-0.0.8\n",
            "Mounted at /content/drive\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-1-0d35c53d087b>:261: UserWarning: The palette list has more values (4) than needed (3), which may not be intended.\n",
            "  sns.violinplot(data=df, x=group_column, y=y_column, palette=colors, linewidth=LINE_WIDTH)\n",
            "ERROR:root:Error creating parallel coordinates plot: 'DataFrame' object has no attribute 'unique'\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Execution completed successfully - Subgroup Discovery Enhanced Notebook.\n"
          ]
        }
      ],
      "source": [
        "!pip install pandas matplotlib seaborn networkx shap scikit-learn plotly scipy\n",
        "# -*- coding: utf-8 -*-\n",
        "\"\"\"Targeted Anxiety Intervention Analysis with Subgroup Discovery\n",
        "\n",
        "This notebook enhances the MoE framework to incorporate subgroup discovery\n",
        "techniques. It aims to identify specific subgroups within intervention groups\n",
        "that show particularly strong or weak responses to the intervention. This\n",
        "allows for a more targeted analysis of intervention effectiveness and\n",
        "personalized insights.\n",
        "\n",
        "Workflow:\n",
        "1. Data Loading and Validation: Load synthetic anxiety intervention data, validate its structure, content, and data types. Handle potential errors gracefully.\n",
        "2. Data Preprocessing: One-hot encode the group column and scale numerical features.\n",
        "3. Subgroup Discovery: Implement a flexible subgroup discovery method to identify response-based subgroups.\n",
        "4. SHAP Value Analysis: Quantify feature importance within discovered subgroups.\n",
        "5. Data Visualization: Generate KDE, Violin, Parallel Coordinates, and Hypergraph plots, highlighting subgroups.\n",
        "6. Statistical Summary: Perform bootstrap analysis and generate summary statistics for subgroups.\n",
        "7. LLM Insights Report: Synthesize findings using Grok, Claude, and Grok-Enhanced, emphasizing subgroup-specific insights, validating LLM outputs, and handling potential LLM API errors.\n",
        "\n",
        "Keywords: Subgroup Discovery, Targeted Analysis, Personalized Intervention, Anxiety, LLMs, SHAP, Data Visualization, Machine Learning\n",
        "\"\"\"\n",
        "\n",
        "# Suppress warnings (with caution - better to handle specific warnings)\n",
        "import warnings\n",
        "import logging  # Use logging for more informative error/warning messages\n",
        "\n",
        "warnings.filterwarnings(\"ignore\", category=FutureWarning)\n",
        "warnings.filterwarnings(\"ignore\", category=UserWarning, module=\"plotly\")\n",
        "\n",
        "# Import libraries\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import networkx as nx\n",
        "import shap\n",
        "import os\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.ensemble import RandomForestRegressor\n",
        "import numpy as np\n",
        "from io import StringIO\n",
        "import plotly.express as px\n",
        "from scipy.stats import bootstrap\n",
        "from matplotlib.colors import LinearSegmentedColormap\n",
        "\n",
        "# --- Setup Logging ---\n",
        "logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\n",
        "\n",
        "# Google Colab environment check\n",
        "try:\n",
        "    from google.colab import drive\n",
        "\n",
        "    drive.mount(\"/content/drive\")\n",
        "    COLAB_ENV = True\n",
        "except ImportError:\n",
        "    COLAB_ENV = False\n",
        "    logging.info(\"Not running in Google Colab environment.\")\n",
        "\n",
        "# Constants\n",
        "OUTPUT_PATH = \"./output_anxiety_subgroup_discovery/\" if not COLAB_ENV else \"/content/drive/MyDrive/output_anxiety_subgroup_discovery/\"\n",
        "PARTICIPANT_ID_COLUMN = \"participant_id\"\n",
        "GROUP_COLUMN = \"group\"  # Keep this for the initial loading and validation\n",
        "ANXIETY_PRE_COLUMN = \"anxiety_pre\"\n",
        "ANXIETY_POST_COLUMN = \"anxiety_post\"\n",
        "MODEL_GROK_NAME = \"grok-base\"\n",
        "MODEL_CLAUDE_NAME = \"claude-3.7-sonnet\"\n",
        "MODEL_GROK_ENHANCED_NAME = \"grok-enhanced\"\n",
        "LINE_WIDTH = 2.5\n",
        "BOOTSTRAP_RESAMPLES = 500\n",
        "\n",
        "# Placeholder API Keys (Security Warning)\n",
        "GROK_API_KEY = \"YOUR_GROK_API_KEY\"  # Placeholder\n",
        "CLAUDE_API_KEY = \"YOUR_CLAUDE_API_KEY\"  # Placeholder\n",
        "\n",
        "\n",
        "# --- Functions ---\n",
        "def create_output_directory(path):\n",
        "    \"\"\"Creates the output directory if it doesn't exist, handling errors.\"\"\"\n",
        "    try:\n",
        "        os.makedirs(path, exist_ok=True)\n",
        "        logging.info(f\"Output directory created/exists: {path}\")\n",
        "        return True\n",
        "    except OSError as e:\n",
        "        logging.error(f\"Failed to create output directory: {path}. Error: {e}\")\n",
        "        return False\n",
        "\n",
        "def load_data_from_synthetic_string(csv_string):\n",
        "    \"\"\"Loads data from a synthetic CSV string, handling errors.\"\"\"\n",
        "    try:\n",
        "        csv_file = StringIO(csv_string)\n",
        "        df = pd.read_csv(csv_file)\n",
        "        logging.info(f\"Data loaded successfully. First 5 rows:\\n{df.head()}\")\n",
        "        logging.info(f\"Data types:\\n{df.dtypes}\")\n",
        "        return df\n",
        "    except pd.errors.ParserError as e:\n",
        "        logging.error(f\"Error parsing CSV data: {e}\")\n",
        "        return None\n",
        "    except Exception as e:\n",
        "        logging.error(f\"Error loading data: {e}\")\n",
        "        return None\n",
        "\n",
        "def validate_dataframe(df, required_columns):\n",
        "    \"\"\"Validates the DataFrame against required columns and data types, handling errors.\"\"\"\n",
        "    if df is None:\n",
        "        logging.error(\"DataFrame is None. Cannot validate.\")\n",
        "        return False\n",
        "\n",
        "    missing_columns = [col for col in required_columns if col not in df.columns]\n",
        "    if missing_columns:\n",
        "        logging.error(f\"Missing columns: {missing_columns}\")\n",
        "        return False\n",
        "\n",
        "    for col in required_columns:\n",
        "        if col != PARTICIPANT_ID_COLUMN and col != GROUP_COLUMN:\n",
        "            if not pd.api.types.is_numeric_dtype(df[col]):\n",
        "                logging.error(f\"Non-numeric values found in column: {col}\")\n",
        "                return False\n",
        "\n",
        "    if df[PARTICIPANT_ID_COLUMN].duplicated().any():\n",
        "        logging.error(\"Duplicate participant IDs found.\")\n",
        "        return False\n",
        "\n",
        "    valid_groups = [\"Group A\", \"Group B\", \"Control\"]\n",
        "    if not df[GROUP_COLUMN].isin(valid_groups).all():\n",
        "        logging.error(f\"Invalid group labels found.  Must be one of: {valid_groups}\")\n",
        "        return False\n",
        "\n",
        "    for col in [ANXIETY_PRE_COLUMN, ANXIETY_POST_COLUMN]:\n",
        "        if df[col].min() < 0 or df[col].max() > 10:\n",
        "            logging.error(f\"Anxiety scores in column '{col}' are out of range (0-10).\")\n",
        "            return False\n",
        "\n",
        "    logging.info(\"DataFrame validation successful.\")\n",
        "    return True\n",
        "\n",
        "def analyze_text_with_llm(text, model_name):  # Placeholder LLM analysis\n",
        "    \"\"\"Placeholder for LLM analysis.  Replace with actual API calls.\"\"\"\n",
        "    text_lower = text.lower()\n",
        "    logging.info(f\"Calling {model_name} with text: {text[:50]}...\")  # Log first 50 chars\n",
        "\n",
        "    if model_name == MODEL_GROK_NAME:\n",
        "        if \"subgroup analysis\" in text_lower:\n",
        "            return \"Grok-base: Subgroup analysis reveals varied responses to the intervention, with some subgroups showing significant improvement while others show minimal change.\"\n",
        "        elif \"shap summary\" in text_lower:\n",
        "            return \"Grok-base: SHAP values highlight feature importance across subgroups, indicating that pre-anxiety is a strong predictor of post-anxiety in all subgroups, but group membership has varying effects.\"\n",
        "        else:\n",
        "            return f\"Grok-base: General analysis on '{text}'.\"\n",
        "    elif model_name == MODEL_CLAUDE_NAME:\n",
        "        if \"subgroup analysis\" in text_lower:\n",
        "            return \"Claude 3.7: Subgroup discovery shows distinct patterns of response to the intervention, identifying groups with strong, weak, and typical responses based on pre- and post-anxiety levels.\"\n",
        "        elif \"violin plot\" in text_lower:\n",
        "            return \"Claude 3.7: Violin plots detail subgroup distributions, clearly showing the differences in anxiety levels and variability between the identified subgroups.\"\n",
        "        else:\n",
        "            return f\"Claude 3.7: Enhanced subgroup analysis on '{text}'.\"\n",
        "    elif model_name == MODEL_GROK_ENHANCED_NAME:\n",
        "        if \"subgroup analysis\" in text_lower:\n",
        "            return \"Grok-Enhanced: Subgroup analysis provides nuanced insights into targeted interventions, revealing specific characteristics of participants who respond differently to the intervention.\"\n",
        "        elif \"parallel coordinates\" in text_lower:\n",
        "            return \"Grok-Enhanced: Parallel coordinates visualize subgroup-specific trajectories, showing how individual participants within each subgroup change from pre- to post-intervention anxiety levels.\"\n",
        "        else:\n",
        "            return f\"Grok-Enhanced: In-depth subgroup-focused analysis on '{text}'.\"\n",
        "    return f\"Model '{model_name}' not supported.\"\n",
        "\n",
        "def scale_data(df, columns):\n",
        "    \"\"\"Scales specified columns of the DataFrame using MinMaxScaler, handling errors.\"\"\"\n",
        "    try:\n",
        "        scaler = MinMaxScaler()\n",
        "        df[columns] = scaler.fit_transform(df[columns])\n",
        "        logging.info(f\"Data scaled successfully. Description:\\n{df[columns].describe()}\")\n",
        "        return df\n",
        "    except Exception as e:\n",
        "        logging.error(f\"Error scaling data: {e}\")\n",
        "        return None  # Return None on error\n",
        "\n",
        "def discover_subgroups(df, encoded_group_cols, output_path):\n",
        "    \"\"\"Identifies subgroups based on intervention response, handling errors.\n",
        "\n",
        "    Args:\n",
        "        df: DataFrame with one-hot encoded group columns.\n",
        "        encoded_group_cols: List of the one-hot encoded group column names.\n",
        "        output_path: Path for output (not used here, but good practice).\n",
        "\n",
        "    Returns:\n",
        "        DataFrame with 'response_level' column, and subgroup description.  Returns\n",
        "        (None, error_message) on failure.\n",
        "    \"\"\"\n",
        "    try:\n",
        "        df['response_level'] = 'typical'  # Default response level\n",
        "\n",
        "        # Construct conditions using the encoded columns\n",
        "        for group_col in encoded_group_cols:\n",
        "            if 'Group A' in group_col:  # Check if this encoded column represents Group A\n",
        "                # Strong responders in Group A: post-anxiety is less than the *overall* mean of pre-anxiety\n",
        "                df.loc[(df[group_col] == 1) & (df[ANXIETY_POST_COLUMN] < df[ANXIETY_PRE_COLUMN].mean()), 'response_level'] = 'strong'\n",
        "            elif 'Group B' in group_col:  # Check if this encoded column represents Group B\n",
        "                # Weak responders in Group B: post-anxiety is *greater* than the *overall* mean of pre-anxiety\n",
        "                df.loc[(df[group_col] == 1) & (df[ANXIETY_POST_COLUMN] > df[ANXIETY_PRE_COLUMN].mean()), 'response_level'] = 'weak'\n",
        "\n",
        "        subgroup_desc = (\n",
        "            \"Subgroups identified based on response to intervention:\\n\"\n",
        "            \"- Strong Responders (Group A, anxiety_post < mean(anxiety_pre)):\\n\"\n",
        "            \"  Participants in Group A showing a strong decrease in post-intervention anxiety.\\n\"\n",
        "            \"- Weak Responders (Group B, anxiety_post > mean(anxiety_pre)):\\n\"\n",
        "            \"  Participants in Group B showing a weak or no decrease in post-intervention anxiety.\\n\"\n",
        "            \"- Typical Responders: Participants not classified as strong or weak responders.\\n\"\n",
        "        )\n",
        "\n",
        "        logging.info(f\"Subgroup Discovery Placeholder Output:\\n{subgroup_desc}\")\n",
        "        logging.info(f\"Response level value counts:\\n{df['response_level'].value_counts()}\")\n",
        "        return df, subgroup_desc\n",
        "\n",
        "    except Exception as e:\n",
        "        logging.error(f\"Error during subgroup discovery: {e}\")\n",
        "        return None, str(e)\n",
        "\n",
        "def calculate_shap_values(df, feature_columns, target_column, output_path):\n",
        "    \"\"\"Calculates SHAP values using a RandomForestRegressor, handling errors.\"\"\"\n",
        "    try:\n",
        "        model_rf = RandomForestRegressor(random_state=42).fit(df[feature_columns], df[target_column]) # Added random_state\n",
        "        explainer = shap.TreeExplainer(model_rf)\n",
        "        shap_values = explainer.shap_values(df[feature_columns])\n",
        "\n",
        "        plt.figure(figsize=(10, 8))\n",
        "        plt.style.use('dark_background')\n",
        "        shap.summary_plot(shap_values, df[feature_columns], show=False, color_bar=True)\n",
        "        plt.savefig(os.path.join(output_path, 'shap_summary.png'))\n",
        "        plt.close()\n",
        "        logging.info(f\"SHAP summary plot saved to {output_path}\")\n",
        "        return f\"SHAP summary for features {feature_columns} predicting {target_column}\"\n",
        "    except Exception as e:\n",
        "        logging.error(f\"Error calculating SHAP values: {e}\")\n",
        "        return \"Error calculating SHAP values.\"\n",
        "\n",
        "def create_kde_plot(df, column1, column2, output_path, colors):\n",
        "    \"\"\"Creates a KDE plot of two columns, handling errors.\"\"\"\n",
        "    try:\n",
        "        plt.figure(figsize=(10, 6))\n",
        "        plt.style.use('dark_background')\n",
        "        sns.kdeplot(\n",
        "            data=df[column1], color=colors[0], label=column1.capitalize(), linewidth=LINE_WIDTH\n",
        "        )\n",
        "        sns.kdeplot(\n",
        "            data=df[column2], color=colors[1], label=column2.capitalize(), linewidth=LINE_WIDTH\n",
        "        )\n",
        "        plt.title(\"KDE Plot of Anxiety Levels\", fontsize=16, color=\"white\")\n",
        "        plt.legend(facecolor=\"black\", edgecolor=\"white\", labelcolor=\"white\")\n",
        "        plt.grid(alpha=0.2, linestyle='--')\n",
        "        plt.tight_layout()\n",
        "        plt.savefig(os.path.join(output_path, \"kde_plot.png\"))\n",
        "        plt.close()\n",
        "\n",
        "        return f\"KDE plot visualizing distributions of {column1} and {column2}\"\n",
        "    except Exception as e:\n",
        "        logging.error(f\"Error creating KDE plot: {e}\")\n",
        "        return \"Error creating KDE plot.\"\n",
        "\n",
        "def create_violin_plot(df, group_column, y_column, output_path, colors):\n",
        "    \"\"\"Creates a violin plot, handling errors.\"\"\"\n",
        "    try:\n",
        "        plt.figure(figsize=(10, 6))\n",
        "        plt.style.use('dark_background')\n",
        "        sns.violinplot(data=df, x=group_column, y=y_column, palette=colors, linewidth=LINE_WIDTH)\n",
        "        plt.title('Violin Plot of Anxiety Distribution by Group', color='white')\n",
        "        plt.savefig(os.path.join(output_path, 'violin_plot.png'))\n",
        "        plt.close()\n",
        "        logging.info(f\"Violin plot saved to {output_path}\")\n",
        "        return f\"Violin plot showing {y_column} across {group_column}\"\n",
        "    except Exception as e:\n",
        "        logging.error(f\"Error creating violin plot: {e}\")\n",
        "        return \"Error creating violin plot.\"\n",
        "\n",
        "def create_parallel_coordinates_plot(df, group_column, anxiety_pre_column, anxiety_post_column, output_path, colors):\n",
        "    \"\"\"Creates a parallel coordinates plot, handling errors.\"\"\"\n",
        "    try:\n",
        "        plot_df = df[[group_column, anxiety_pre_column, anxiety_post_column, 'response_level']].copy()\n",
        "        unique_groups = plot_df['response_level'].unique()\n",
        "        group_color_map = {group: colors[i % len(colors)] for i, group in enumerate(unique_groups)}\n",
        "        plot_df['color'] = plot_df['response_level'].map(group_color_map)\n",
        "        fig = px.parallel_coordinates(\n",
        "            plot_df,\n",
        "            color='color',\n",
        "            dimensions=[anxiety_pre_column, anxiety_post_column],\n",
        "            title=\"Anxiety Levels: Pre- vs Post-Intervention by Response Subgroup\",\n",
        "            color_continuous_scale=px.colors.sequential.Viridis\n",
        "        )\n",
        "        fig.update_layout(plot_bgcolor='black', paper_bgcolor='black', font_color='white', title_font_size=16)\n",
        "        fig.write_image(os.path.join(output_path, 'parallel_coordinates_plot_subgroups.png'))\n",
        "        logging.info(f\"Parallel coordinates plot saved to {output_path}\")\n",
        "        return \"Parallel coordinates plot of anxiety pre vs post intervention by response subgroup\"\n",
        "    except Exception as e:\n",
        "        logging.error(f\"Error creating parallel coordinates plot: {e}\")\n",
        "        return \"Error creating parallel coordinates plot.\"\n",
        "\n",
        "def visualize_hypergraph(df, anxiety_pre_column, anxiety_post_column, output_path, colors):\n",
        "    \"\"\"Visualizes a hypergraph, handling errors.\"\"\"\n",
        "    try:\n",
        "        G = nx.Graph()\n",
        "        participant_ids = df[PARTICIPANT_ID_COLUMN].tolist()\n",
        "        G.add_nodes_from(participant_ids, bipartite=0)\n",
        "        feature_sets = {\n",
        "            \"anxiety_pre\": df[PARTICIPANT_ID_COLUMN][df[anxiety_pre_column] > df[anxiety_pre_column].mean()].tolist(),\n",
        "            \"anxiety_post\": df[PARTICIPANT_ID_COLUMN][df[anxiety_post_column] > df[anxiety_post_column].mean()].tolist(),\n",
        "            \"strong_response\": df[PARTICIPANT_ID_COLUMN][df['response_level'] == 'strong'].tolist()\n",
        "        }\n",
        "        feature_nodes = list(feature_sets.keys())\n",
        "        G.add_nodes_from(feature_nodes, bipartite=1)\n",
        "        for feature, participants in feature_sets.items():\n",
        "            for participant in participants:\n",
        "                G.add_edge(participant, feature)\n",
        "        pos = nx.bipartite_layout(G, participant_ids)\n",
        "        color_map = [colors[0] if node in participant_ids else colors[1] for node in G]\n",
        "        plt.figure(figsize=(12, 10))\n",
        "        plt.style.use('dark_background')\n",
        "        nx.draw(G, pos, with_labels=True, node_color=color_map, font_color=\"white\", edge_color=\"gray\",\n",
        "                width=LINE_WIDTH, node_size=700, font_size=10)\n",
        "        plt.title(\"Hypergraph Representation of Anxiety Patterns with Subgroups\", color=\"white\")\n",
        "        plt.savefig(os.path.join(output_path, \"hypergraph_subgroups.png\"))\n",
        "        plt.close()\n",
        "        logging.info(f\"Hypergraph saved to {output_path}\")\n",
        "        return \"Hypergraph visualizing participant relationships, highlighting response subgroups\"\n",
        "    except Exception as e:\n",
        "        logging.error(f\"Error creating hypergraph: {e}\")\n",
        "        return \"Error creating hypergraph.\"\n",
        "\n",
        "def perform_bootstrap(data, statistic, n_resamples=BOOTSTRAP_RESAMPLES):\n",
        "    \"\"\"Performs bootstrap analysis, handling errors.\"\"\"\n",
        "    try:\n",
        "        bootstrap_result = bootstrap((data,), statistic, n_resamples=n_resamples, method='percentile', random_state=42) # Added random_state\n",
        "        logging.info(f\"Bootstrap CI: {bootstrap_result.confidence_interval}\")\n",
        "        return bootstrap_result.confidence_interval\n",
        "    except Exception as e:\n",
        "        logging.error(f\"Error performing bootstrap: {e}\")\n",
        "        return None\n",
        "\n",
        "def save_summary(df, bootstrap_ci, output_path):\n",
        "    \"\"\"Saves summary statistics, handling errors.\"\"\"\n",
        "    try:\n",
        "        summary_text = (\n",
        "            df.describe().to_string() +\n",
        "            f\"\\nBootstrap CI for anxiety_post mean (all participants): {bootstrap_ci}\\n\\n\"\n",
        "            f\"Summary by Response Subgroup:\\n\"\n",
        "            f\"{df.groupby('response_level')[[ANXIETY_PRE_COLUMN, ANXIETY_POST_COLUMN]].describe().to_string()}\"\n",
        "        )\n",
        "        with open(os.path.join(output_path, 'summary.txt'), 'w') as f:\n",
        "            f.write(summary_text)\n",
        "        logging.info(f\"Summary statistics saved to {output_path}\")\n",
        "        return summary_text\n",
        "    except Exception as e:\n",
        "        logging.error(f\"Error saving summary: {e}\")\n",
        "        return \"Error saving summary.\"\n",
        "\n",
        "def generate_insights_report(summary_stats_text, subgroup_desc, shap_analysis_info, kde_plot_desc, violin_plot_desc, parallel_coords_desc, hypergraph_desc, output_path):\n",
        "    \"\"\"Generates a combined insights report using (simulated) LLM calls.\"\"\"\n",
        "\n",
        "    try:\n",
        "        grok_insights = (\n",
        "                analyze_text_with_llm(f\"Analyze summary statistics including subgroup analysis:\\n{summary_stats_text}\",\n",
        "                                      MODEL_GROK_NAME) + \"\\n\\n\" +\n",
        "                analyze_text_with_llm(f\"Interpret SHAP summary for subgroups: {shap_analysis_info}\", MODEL_GROK_NAME) + \"\\n\\n\" +\n",
        "                analyze_text_with_llm(f\"Describe the identified subgroups: {subgroup_desc}\", MODEL_GROK_NAME) + \"\\n\\n\"\n",
        "        )\n",
        "        claude_insights = (\n",
        "                analyze_text_with_llm(f\"Interpret KDE plot for subgroups: {kde_plot_desc}\", MODEL_CLAUDE_NAME) + \"\\n\\n\" +\n",
        "                analyze_text_with_llm(f\"Interpret Violin plot for subgroups: {violin_plot_desc}\",\n",
        "                                      MODEL_CLAUDE_NAME) + \"\\n\\n\" +\n",
        "                analyze_text_with_llm(f\"Interpret Parallel Coordinates Plot for subgroups: {parallel_coords_desc}\",\n",
        "                                      MODEL_CLAUDE_NAME) + \"\\n\\n\" +\n",
        "                analyze_text_with_llm(f\"Interpret Hypergraph highlighting subgroups: {hypergraph_desc}\",\n",
        "                                      MODEL_CLAUDE_NAME) + \"\\n\\n\"\n",
        "        )\n",
        "        grok_enhanced_insights = analyze_text_with_llm(\n",
        "            f\"Provide enhanced insights on anxiety intervention effectiveness based on subgroup analysis, SHAP, and Parallel Coordinates, focusing on differences between subgroups.\",\n",
        "            MODEL_GROK_ENHANCED_NAME)\n",
        "\n",
        "        combined_insights = f\"\"\"\n",
        "    Combined Insights Report: Anxiety Intervention Analysis with Subgroup Discovery\n",
        "\n",
        "    Grok-base Analysis:\n",
        "    {grok_insights}\n",
        "\n",
        "    Claude 3.7 Sonnet Analysis:\n",
        "    {claude_insights}\n",
        "\n",
        "    Grok-Enhanced Analysis (Subgroup Focused):\n",
        "    {grok_enhanced_insights}\n",
        "\n",
        "    Synthesized Summary:\n",
        "    This report synthesizes insights from Grok-base, Claude 3.7 Sonnet, and Grok-Enhanced, focusing on subgroup discovery to refine the analysis of anxiety intervention effectiveness.  Grok-base provides a statistical overview, initial subgroup interpretations, and feature importances across subgroups, noting the strong influence of pre-anxiety. Claude 3.7 Sonnet details visual patterns and distributions, highlighting subgroup-specific variations and the shift towards lower anxiety in the 'strong responders' subgroup. Grok-Enhanced, with a focus on subgroups, delivers nuanced interpretations and actionable recommendations tailored to different response patterns, revealing specific characteristics of participants. The combined expert analyses, enhanced by subgroup discovery, provide a targeted and personalized understanding of the anxiety intervention, enabling tailored strategies for different responder profiles. The identified subgroups ('strong responders', 'weak responders', and 'typical responders') show distinct patterns in their response to the intervention.\n",
        "    \"\"\"\n",
        "        with open(os.path.join(output_path, 'insights.txt'), 'w') as f:\n",
        "            f.write(combined_insights)\n",
        "        logging.info(f\"Insights report saved to {output_path}\")\n",
        "        return \"Insights report generated successfully.\"\n",
        "\n",
        "    except Exception as e:\n",
        "        logging.error(f\"Error generating insights report: {e}\")\n",
        "        return \"Error generating insights report.\"\n",
        "\n",
        "# --- Main Script ---\n",
        "if __name__ == \"__main__\":\n",
        "    # Create output directory\n",
        "    if not create_output_directory(OUTPUT_PATH):\n",
        "        exit()\n",
        "\n",
        "    # Synthetic dataset (small, embedded in code)\n",
        "    synthetic_dataset = \"\"\"\n",
        "participant_id,group,anxiety_pre,anxiety_post\n",
        "P001,Group A,4,2\n",
        "P002,Group A,3,1\n",
        "P003,Group A,5,3\n",
        "P004,Group B,6,5\n",
        "P005,Group B,5,4\n",
        "P006,Group B,7,6\n",
        "P007,Control,3,3\n",
        "P008,Control,4,4\n",
        "P009,Control,2,2\n",
        "P010,Control,5,5\n",
        "\"\"\"\n",
        "    # Load and validate data\n",
        "    df = load_data_from_synthetic_string(synthetic_dataset)\n",
        "    if df is None:\n",
        "        exit()\n",
        "\n",
        "    required_columns = [PARTICIPANT_ID_COLUMN, GROUP_COLUMN, ANXIETY_PRE_COLUMN, ANXIETY_POST_COLUMN]\n",
        "    if not validate_dataframe(df, required_columns):\n",
        "        exit()\n",
        "\n",
        "    # Keep a copy of the original dataframe for visualizations\n",
        "    df_original = df.copy()\n",
        "\n",
        "    # One-hot encode 'group' *before* subgroup discovery and scaling\n",
        "    df = pd.get_dummies(df, columns=[GROUP_COLUMN], prefix=GROUP_COLUMN, drop_first=False)  # One-hot encode, keep all groups\n",
        "    encoded_group_cols = [col for col in df.columns if col.startswith(f\"{GROUP_COLUMN}_\")]\n",
        "\n",
        "    # Scale data\n",
        "    df = scale_data(df, [ANXIETY_PRE_COLUMN, ANXIETY_POST_COLUMN] + encoded_group_cols)\n",
        "    if df is None:\n",
        "        exit()\n",
        "\n",
        "    # Subgroup Discovery (using the encoded group columns)\n",
        "    df, subgroup_desc = discover_subgroups(df, encoded_group_cols, OUTPUT_PATH)\n",
        "    if df is None:\n",
        "        exit()\n",
        "\n",
        "    # SHAP analysis (using the encoded group columns)\n",
        "    shap_feature_columns = encoded_group_cols + [ANXIETY_PRE_COLUMN]\n",
        "    shap_analysis_info = calculate_shap_values(df.copy(), shap_feature_columns, ANXIETY_POST_COLUMN,\n",
        "                                                OUTPUT_PATH)\n",
        "\n",
        "    # Visualization colors\n",
        "    neon_colors = [\"#FF00FF\", \"#00FFFF\", \"#FFFF00\", \"#00FF00\"]\n",
        "\n",
        "    # Create visualizations (using df_original for plots that need original group labels)\n",
        "    kde_plot_desc = create_kde_plot(\n",
        "        df, ANXIETY_PRE_COLUMN, ANXIETY_POST_COLUMN, OUTPUT_PATH, neon_colors[:2]\n",
        "    )  # Use scaled, encoded df\n",
        "    violin_plot_desc = create_violin_plot(\n",
        "        df, 'response_level', ANXIETY_POST_COLUMN, OUTPUT_PATH, neon_colors\n",
        "    )  # Use the new 'response_level' column\n",
        "    parallel_coords_desc = create_parallel_coordinates_plot(\n",
        "        df, 'response_level', ANXIETY_PRE_COLUMN, ANXIETY_POST_COLUMN, OUTPUT_PATH, neon_colors\n",
        "    )  # Use 'response_level'\n",
        "    hypergraph_desc = visualize_hypergraph(\n",
        "        df, ANXIETY_PRE_COLUMN, ANXIETY_POST_COLUMN, OUTPUT_PATH, neon_colors[:2]\n",
        "    )  # Use the modified df\n",
        "\n",
        "    # Bootstrap analysis\n",
        "    bootstrap_ci = perform_bootstrap(df[ANXIETY_POST_COLUMN], np.mean)\n",
        "\n",
        "    # Save summary statistics\n",
        "    summary_stats_text = save_summary(df, bootstrap_ci, OUTPUT_PATH)\n",
        "\n",
        "    # Generate insights report\n",
        "    generate_insights_report(summary_stats_text, subgroup_desc, shap_analysis_info, kde_plot_desc, violin_plot_desc, parallel_coords_desc, hypergraph_desc, OUTPUT_PATH)\n",
        "\n",
        "    print(\"Execution completed successfully - Subgroup Discovery Enhanced Notebook.\")"
      ]
    }
  ]
}