Add InfiniteBenchSum scenario and run spec #3409
Conversation
|
|
||
|
|
||
| @run_spec_function("infinite_bench_sum") | ||
| def get_infinite_bench_sum_spec(word_lower_bound: float = 0.0, word_upper_bound: float = 100e6) -> RunSpec: |
There was a problem hiding this comment.
1e8 instead of 100e6 (also isn't 1e7 enough?)
There was a problem hiding this comment.
yes 1e7 is enough; re 1e8 vs 100e6, was meant to make it easier to understand (100e6=100M, 30e3=30k, etc), but am fine with both.
Currently: changed to 1e7 in the latest change
| scenario_spec = ScenarioSpec( | ||
| class_name="helm.benchmark.scenarios.infinite_bench_sum_scenario.InfiniteBenchSumScenario", | ||
| args={ | ||
| "word_lower_bound": word_lower_bound, |
There was a problem hiding this comment.
num_words_lower_bound or min_num_words
There was a problem hiding this comment.
Changed to min_num_words
| class_name="helm.benchmark.scenarios.infinite_bench_sum_scenario.InfiniteBenchSumScenario", | ||
| args={ | ||
| "word_lower_bound": word_lower_bound, | ||
| "word_upper_bound": word_upper_bound, |
There was a problem hiding this comment.
num_words_upper_bound or max_num_words
There was a problem hiding this comment.
Changed to max_num_words
| class InfiniteBenchSumScenario(Scenario): | ||
| """InfiniteBenchSum | ||
|
|
||
| InfiniteBenchbenchmark tailored for evaluating the capabilities of language models to process, |
|
|
||
|
|
||
| class InfiniteBenchSumScenario(Scenario): | ||
| """InfiniteBenchSum |
There was a problem hiding this comment.
Is there a space between InfiniteBench and Sum?
| dataset = dataset.map(lambda example: {"prompt": example["context"] + "\n\n" + example["input"]}) | ||
| dataset = dataset.map(lambda example: {"prompt_wc": len(example["prompt"].split())}) | ||
| dataset = dataset.filter(lambda example: self.word_lower_bound <= example["prompt_wc"] <= self.word_upper_bound) |
There was a problem hiding this comment.
can you just do this chained
(
dataset.map()
.map()
.filter()
)
There was a problem hiding this comment.
Yes, changed to this
|
|
||
| assert isinstance(dataset, Dataset) | ||
|
|
||
| dataset = dataset.map(lambda example: {"prompt": example["context"] + "\n\n" + example["input"]}) |
| assert isinstance(dataset, Dataset) | ||
|
|
||
| dataset = dataset.map(lambda example: {"prompt": example["context"] + "\n\n" + example["input"]}) | ||
| dataset = dataset.map(lambda example: {"prompt_wc": len(example["prompt"].split())}) |
There was a problem hiding this comment.
def count_words:
return len(re.split(r"\s+", text.strip()))
then do
{"prompt_wc": count_words(example["context"]) + count_words(example["input"])}
| ) | ||
|
|
||
| # Load the dataset with the specified features | ||
| dataset = load_dataset("xinrongzhang2022/InfiniteBench", split="longbook_sum_eng", features=ft) |
There was a problem hiding this comment.
pin revision revision="90f0394333616266d9fe85824ceaf505093cbaa5"
No description provided.