model_subset_selection_guide.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"https://colab.research.google.com/github/ryoungj/ObsScaling/blob/main/model_subset_selection_eval.ipynb\">\n",
    "  <img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/>\n",
    "</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Subset Selection (Guideline)\n",
    "\n",
    "This notebook provides a guideline with minimal examples to select a subset of available models with optimal experimental design principle that minimize the evaluation cost while maintaining the prediction performance."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Colab specific setup: uncomment the following lines in colab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! git clone https://github.com/ryoungj/ObsScaling\n",
    "# %cd ObsScaling\n",
    "# ! pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pickle\n",
    "import json\n",
    "import re\n",
    "import copy\n",
    "\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload \n",
    "%autoreload 2\n",
    "%config InlineBackend.figure_format = 'retina'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Guidelines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 1: Load benchmark eval results for LLMs to select from"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_llm_benchmark_eval = load_base_llm_benchmark_eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Step 2: Specify model selection arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## This is an illustrative example \n",
    "## Specify your own arguments based on your own data and needs\n",
    "DEFAULT_SELECTION_KWARGS = {\n",
    "    \"num_model_budgets\": [4, 8, 12, 16, 20, 24, 28, 32, 36],  # number of model budgets\n",
    "    \"max_family_to_search\": 10,  # maximum number of model families to brute search\n",
    "    \"include_family\": \"Llama-2\",  # always include Llama-2 in the selection as it is the most widely used model family\n",
    "    \"num_pc_for_select\": 3,  # number of PCs to compute the variance for model selection\n",
    "    \"all_families_for_select\": EVAL_BASE_MODEL_FAMILIES,  # all model families to consider for model selection\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 3: Selecting model set under additional budgets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Selecting models from all available models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total:  431909\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "431909it [00:17, 23995.38it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> Num model budegt: 4\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -1365.30\n",
      " \t Model family (2): Mistral, Llama-2\n",
      " \t Models (4): meta-llama/Llama-2-7b-hf, meta-llama/Llama-2-13b-hf, meta-llama/Llama-2-70b-hf, mistralai/Mistral-7B-v0.1\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 8\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -37.43\n",
      " \t Model family (4): Mixtral, Phi, MPT, Llama-2\n",
      " \t Models (8): meta-llama/Llama-2-7b-hf, meta-llama/Llama-2-13b-hf, meta-llama/Llama-2-70b-hf, mistralai/Mixtral-8x7B-v0.1, microsoft/phi-2, microsoft/phi-1_5, mosaicml/mpt-30b, mosaicml/mpt-7b\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 12\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -16.93\n",
      " \t Model family (4): Llama-3, Falcon, DeepSeek-Coder, Llama-2\n",
      " \t Models (12): meta-llama/Llama-2-7b-hf, meta-llama/Llama-2-13b-hf, meta-llama/Llama-2-70b-hf, meta-llama/Meta-Llama-3-70B, meta-llama/Meta-Llama-3-8B, tiiuae/falcon-180B, tiiuae/falcon-40b, tiiuae/falcon-7b, tiiuae/falcon-rw-1b, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base, deepseek-ai/deepseek-coder-33b-base\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 16\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -12.70\n",
      " \t Model family (6): Llama-3, Falcon, Phi, MPT, DeepSeek-Coder, Llama-2\n",
      " \t Models (16): meta-llama/Llama-2-7b-hf, meta-llama/Llama-2-13b-hf, meta-llama/Llama-2-70b-hf, meta-llama/Meta-Llama-3-70B, meta-llama/Meta-Llama-3-8B, tiiuae/falcon-180B, tiiuae/falcon-40b, tiiuae/falcon-7b, tiiuae/falcon-rw-1b, microsoft/phi-2, microsoft/phi-1_5, mosaicml/mpt-30b, mosaicml/mpt-7b, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base, deepseek-ai/deepseek-coder-33b-base\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 20\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -10.25\n",
      " \t Model family (7): Qwen, Mixtral, Falcon, MPT, CodeLlama, DeepSeek-Coder, Llama-2\n",
      " \t Models (20): meta-llama/Llama-2-7b-hf, meta-llama/Llama-2-13b-hf, meta-llama/Llama-2-70b-hf, Qwen/Qwen-72B, Qwen/Qwen-14B, Qwen/Qwen-7B, mistralai/Mixtral-8x7B-v0.1, tiiuae/falcon-180B, tiiuae/falcon-40b, tiiuae/falcon-7b, tiiuae/falcon-rw-1b, mosaicml/mpt-30b, mosaicml/mpt-7b, codellama/CodeLlama-7b-hf, codellama/CodeLlama-13b-hf, codellama/CodeLlama-34b-hf, codellama/CodeLlama-70b-hf, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base, deepseek-ai/deepseek-coder-33b-base\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 24\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -8.35\n",
      " \t Model family (8): Llama-3, Qwen, Falcon, MPT, CodeLlama, StarCoder2, DeepSeek-Coder, Llama-2\n",
      " \t Models (24): meta-llama/Llama-2-7b-hf, meta-llama/Llama-2-13b-hf, meta-llama/Llama-2-70b-hf, meta-llama/Meta-Llama-3-70B, meta-llama/Meta-Llama-3-8B, Qwen/Qwen-72B, Qwen/Qwen-14B, Qwen/Qwen-7B, tiiuae/falcon-180B, tiiuae/falcon-40b, tiiuae/falcon-7b, tiiuae/falcon-rw-1b, mosaicml/mpt-30b, mosaicml/mpt-7b, codellama/CodeLlama-7b-hf, codellama/CodeLlama-13b-hf, codellama/CodeLlama-34b-hf, codellama/CodeLlama-70b-hf, bigcode/starcoder2-15b, bigcode/starcoder2-7b, bigcode/starcoder2-3b, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base, deepseek-ai/deepseek-coder-33b-base\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 28\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -7.21\n",
      " \t Model family (8): Llama, Llama-3, Qwen1.5, Mixtral, Falcon, CodeLlama, DeepSeek-Coder, Llama-2\n",
      " \t Models (28): meta-llama/Llama-2-7b-hf, meta-llama/Llama-2-13b-hf, meta-llama/Llama-2-70b-hf, huggyllama/llama-7b, huggyllama/llama-13b, huggyllama/llama-30b, huggyllama/llama-65b, meta-llama/Meta-Llama-3-70B, meta-llama/Meta-Llama-3-8B, Qwen/Qwen1.5-72B, Qwen/Qwen1.5-32B, Qwen/Qwen1.5-14B, Qwen/Qwen1.5-7B, Qwen/Qwen1.5-4B, Qwen/Qwen1.5-1.8B, Qwen/Qwen1.5-0.5B, mistralai/Mixtral-8x7B-v0.1, tiiuae/falcon-180B, tiiuae/falcon-40b, tiiuae/falcon-7b, tiiuae/falcon-rw-1b, codellama/CodeLlama-7b-hf, codellama/CodeLlama-13b-hf, codellama/CodeLlama-34b-hf, codellama/CodeLlama-70b-hf, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base, deepseek-ai/deepseek-coder-33b-base\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 32\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -6.30\n",
      " \t Model family (10): Llama-3, Qwen, Mixtral, Yi, Falcon, OPT, MPT, CodeLlama, DeepSeek-Coder, Llama-2\n",
      " \t Models (32): meta-llama/Llama-2-7b-hf, meta-llama/Llama-2-13b-hf, meta-llama/Llama-2-70b-hf, meta-llama/Meta-Llama-3-70B, meta-llama/Meta-Llama-3-8B, Qwen/Qwen-72B, Qwen/Qwen-14B, Qwen/Qwen-7B, mistralai/Mixtral-8x7B-v0.1, 01-ai/Yi-6B, 01-ai/Yi-34B, tiiuae/falcon-180B, tiiuae/falcon-40b, tiiuae/falcon-7b, tiiuae/falcon-rw-1b, facebook/opt-6.7b, facebook/opt-1.3b, facebook/opt-350m, facebook/opt-13b, facebook/opt-2.7b, facebook/opt-30b, facebook/opt-125m, facebook/opt-66b, mosaicml/mpt-30b, mosaicml/mpt-7b, codellama/CodeLlama-7b-hf, codellama/CodeLlama-13b-hf, codellama/CodeLlama-34b-hf, codellama/CodeLlama-70b-hf, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base, deepseek-ai/deepseek-coder-33b-base\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 36\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -5.65\n",
      " \t Model family (9): Llama-3, Qwen1.5, Mixtral, Falcon, OPT, CodeLlama, StarCoder, DeepSeek-Coder, Llama-2\n",
      " \t Models (36): meta-llama/Llama-2-7b-hf, meta-llama/Llama-2-13b-hf, meta-llama/Llama-2-70b-hf, meta-llama/Meta-Llama-3-70B, meta-llama/Meta-Llama-3-8B, Qwen/Qwen1.5-72B, Qwen/Qwen1.5-32B, Qwen/Qwen1.5-14B, Qwen/Qwen1.5-7B, Qwen/Qwen1.5-4B, Qwen/Qwen1.5-1.8B, Qwen/Qwen1.5-0.5B, mistralai/Mixtral-8x7B-v0.1, tiiuae/falcon-180B, tiiuae/falcon-40b, tiiuae/falcon-7b, tiiuae/falcon-rw-1b, facebook/opt-6.7b, facebook/opt-1.3b, facebook/opt-350m, facebook/opt-13b, facebook/opt-2.7b, facebook/opt-30b, facebook/opt-125m, facebook/opt-66b, codellama/CodeLlama-7b-hf, codellama/CodeLlama-13b-hf, codellama/CodeLlama-34b-hf, codellama/CodeLlama-70b-hf, bigcode/starcoderbase-1b, bigcode/starcoderbase-3b, bigcode/starcoderbase-7b, bigcode/starcoderbase, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base, deepseek-ai/deepseek-coder-33b-base\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "run_results, select_results = search_subset(base_llm_benchmark_eval, **DEFAULT_SELECTION_KWARGS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Selecting models under additional budget constraints (e.g., sub 7B models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total:  89845\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "89845it [00:19, 4609.46it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> Num model budegt: 4\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -25.61\n",
      " \t Model family (3): Phi, CodeLlama, Llama-2\n",
      " \t Models (4): meta-llama/Llama-2-7b-hf, microsoft/phi-2, microsoft/phi-1_5, codellama/CodeLlama-7b-hf\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 8\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -10.51\n",
      " \t Model family (6): Llama, Qwen, Phi, MPT, DeepSeek-Coder, Llama-2\n",
      " \t Models (8): meta-llama/Llama-2-7b-hf, huggyllama/llama-7b, Qwen/Qwen-7B, microsoft/phi-2, microsoft/phi-1_5, mosaicml/mpt-7b, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 12\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -7.04\n",
      " \t Model family (8): Llama, Qwen, Gemma, Falcon, Phi, MPT, DeepSeek-Coder, Llama-2\n",
      " \t Models (12): meta-llama/Llama-2-7b-hf, huggyllama/llama-7b, Qwen/Qwen-7B, google/gemma-7b, google/gemma-2b, tiiuae/falcon-7b, tiiuae/falcon-rw-1b, microsoft/phi-2, microsoft/phi-1_5, mosaicml/mpt-7b, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 16\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -5.71\n",
      " \t Model family (9): Llama, Qwen1.5, Gemma, Falcon, Phi, MPT, CodeLlama, DeepSeek-Coder, Llama-2\n",
      " \t Models (16): meta-llama/Llama-2-7b-hf, huggyllama/llama-7b, Qwen/Qwen1.5-7B, Qwen/Qwen1.5-4B, Qwen/Qwen1.5-1.8B, Qwen/Qwen1.5-0.5B, google/gemma-7b, google/gemma-2b, tiiuae/falcon-7b, tiiuae/falcon-rw-1b, microsoft/phi-2, microsoft/phi-1_5, mosaicml/mpt-7b, codellama/CodeLlama-7b-hf, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 20\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -5.04\n",
      " \t Model family (10): Llama, Qwen1.5, Yi, Gemma, Phi, OPT, MPT, CodeLlama, DeepSeek-Coder, Llama-2\n",
      " \t Models (20): meta-llama/Llama-2-7b-hf, huggyllama/llama-7b, Qwen/Qwen1.5-7B, Qwen/Qwen1.5-4B, Qwen/Qwen1.5-1.8B, Qwen/Qwen1.5-0.5B, 01-ai/Yi-6B, google/gemma-7b, google/gemma-2b, microsoft/phi-2, microsoft/phi-1_5, facebook/opt-6.7b, facebook/opt-1.3b, facebook/opt-350m, facebook/opt-2.7b, facebook/opt-125m, mosaicml/mpt-7b, codellama/CodeLlama-7b-hf, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 24\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -4.57\n",
      " \t Model family (10): Qwen1.5, Gemma, Falcon, Phi, Pythia, MPT, CodeLlama, StarCoder2, DeepSeek-Coder, Llama-2\n",
      " \t Models (24): meta-llama/Llama-2-7b-hf, Qwen/Qwen1.5-7B, Qwen/Qwen1.5-4B, Qwen/Qwen1.5-1.8B, Qwen/Qwen1.5-0.5B, google/gemma-7b, google/gemma-2b, tiiuae/falcon-7b, tiiuae/falcon-rw-1b, microsoft/phi-2, microsoft/phi-1_5, EleutherAI/pythia-1b-deduped, EleutherAI/pythia-410m-deduped, EleutherAI/pythia-6.9b-deduped, EleutherAI/pythia-2.8b-deduped, EleutherAI/pythia-70m-deduped, EleutherAI/pythia-1.4b-deduped, EleutherAI/pythia-160m-deduped, mosaicml/mpt-7b, codellama/CodeLlama-7b-hf, bigcode/starcoder2-7b, bigcode/starcoder2-3b, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 28\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -4.27\n",
      " \t Model family (10): Qwen1.5, Gemma, Falcon, Phi, Pythia, OPT, MPT, StarCoder2, DeepSeek-Coder, Llama-2\n",
      " \t Models (28): meta-llama/Llama-2-7b-hf, Qwen/Qwen1.5-7B, Qwen/Qwen1.5-4B, Qwen/Qwen1.5-1.8B, Qwen/Qwen1.5-0.5B, google/gemma-7b, google/gemma-2b, tiiuae/falcon-7b, tiiuae/falcon-rw-1b, microsoft/phi-2, microsoft/phi-1_5, EleutherAI/pythia-1b-deduped, EleutherAI/pythia-410m-deduped, EleutherAI/pythia-6.9b-deduped, EleutherAI/pythia-2.8b-deduped, EleutherAI/pythia-70m-deduped, EleutherAI/pythia-1.4b-deduped, EleutherAI/pythia-160m-deduped, facebook/opt-6.7b, facebook/opt-1.3b, facebook/opt-350m, facebook/opt-2.7b, facebook/opt-125m, mosaicml/mpt-7b, bigcode/starcoder2-7b, bigcode/starcoder2-3b, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 32\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -4.15\n",
      " \t Model family (10): Qwen1.5, Gemma, Falcon, Phi, Pythia, GPT-Neo/J, OPT, StarCoder, DeepSeek-Coder, Llama-2\n",
      " \t Models (32): meta-llama/Llama-2-7b-hf, Qwen/Qwen1.5-7B, Qwen/Qwen1.5-4B, Qwen/Qwen1.5-1.8B, Qwen/Qwen1.5-0.5B, google/gemma-7b, google/gemma-2b, tiiuae/falcon-7b, tiiuae/falcon-rw-1b, microsoft/phi-2, microsoft/phi-1_5, EleutherAI/pythia-1b-deduped, EleutherAI/pythia-410m-deduped, EleutherAI/pythia-6.9b-deduped, EleutherAI/pythia-2.8b-deduped, EleutherAI/pythia-70m-deduped, EleutherAI/pythia-1.4b-deduped, EleutherAI/pythia-160m-deduped, EleutherAI/gpt-neo-2.7B, EleutherAI/gpt-neo-1.3B, EleutherAI/gpt-neo-125m, EleutherAI/gpt-j-6b, facebook/opt-6.7b, facebook/opt-1.3b, facebook/opt-350m, facebook/opt-2.7b, facebook/opt-125m, bigcode/starcoderbase-1b, bigcode/starcoderbase-3b, bigcode/starcoderbase-7b, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ">>> Num model budegt: 36\n",
      "\n",
      "### Best configs:\n",
      " \t Object value: -4.15\n",
      " \t Model family (10): Qwen1.5, Gemma, Falcon, Phi, Pythia, GPT-Neo/J, OPT, StarCoder, DeepSeek-Coder, Llama-2\n",
      " \t Models (32): meta-llama/Llama-2-7b-hf, Qwen/Qwen1.5-7B, Qwen/Qwen1.5-4B, Qwen/Qwen1.5-1.8B, Qwen/Qwen1.5-0.5B, google/gemma-7b, google/gemma-2b, tiiuae/falcon-7b, tiiuae/falcon-rw-1b, microsoft/phi-2, microsoft/phi-1_5, EleutherAI/pythia-1b-deduped, EleutherAI/pythia-410m-deduped, EleutherAI/pythia-6.9b-deduped, EleutherAI/pythia-2.8b-deduped, EleutherAI/pythia-70m-deduped, EleutherAI/pythia-1.4b-deduped, EleutherAI/pythia-160m-deduped, EleutherAI/gpt-neo-2.7B, EleutherAI/gpt-neo-1.3B, EleutherAI/gpt-neo-125m, EleutherAI/gpt-j-6b, facebook/opt-6.7b, facebook/opt-1.3b, facebook/opt-350m, facebook/opt-2.7b, facebook/opt-125m, bigcode/starcoderbase-1b, bigcode/starcoderbase-3b, bigcode/starcoderbase-7b, deepseek-ai/deepseek-coder-1.3b-base, deepseek-ai/deepseek-coder-6.7b-base\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## Specify cutoff kwargs to keep models under 7B parameters\n",
    "## You can also do that by pre-filtering the `base_llm_benchmark_eval` based on your own needs\n",
    "CUTOFF_KWARGS = {\n",
    "    \"split_method\": \"cutoff_by_Model Size (B)\",\n",
    "    \"cutoff_threshold\": 7,\n",
    "}\n",
    "\n",
    "SELECTION_KWARGS = {\n",
    "    **DEFAULT_SELECTION_KWARGS,\n",
    "\n",
    "    \"cutoff_kwargs\": CUTOFF_KWARGS,\n",
    "}\n",
    "run_results, select_results = search_subset(base_llm_benchmark_eval, **SELECTION_KWARGS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "agentevalhub",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}