|
| 1 | +import os |
| 2 | +import re |
| 3 | +from typing import Dict, List, Optional |
| 4 | + |
| 5 | +import pandas as pd |
| 6 | + |
| 7 | +from helm.common.general import ensure_file_downloaded, ensure_directory_exists |
| 8 | +from helm.common.optional_dependencies import handle_module_not_found_error |
| 9 | +from helm.benchmark.scenarios.scenario import ( |
| 10 | + Scenario, |
| 11 | + Instance, |
| 12 | + Reference, |
| 13 | + TRAIN_SPLIT, |
| 14 | + TEST_SPLIT, |
| 15 | + CORRECT_TAG, |
| 16 | + Input, |
| 17 | + Output, |
| 18 | +) |
| 19 | + |
| 20 | +try: |
| 21 | + # Needed for pandas.read_excel |
| 22 | + import openpyxl # noqa |
| 23 | +except ModuleNotFoundError as e: |
| 24 | + handle_module_not_found_error(e, ["ibm-enterprise-scenarios"]) |
| 25 | + |
| 26 | + |
| 27 | +class SUMOSumScenario(Scenario): |
| 28 | + """SUMO Web Claims Summarization |
| 29 | +
|
| 30 | + SUMO Web Claims Summarization is a summarization task over the climate subset from the SUMO dataset. |
| 31 | + The task is to write a title based on the article contents. |
| 32 | +
|
| 33 | + Citation: |
| 34 | + @inproceedings{mishra-etal-2020-generating, |
| 35 | + title = "Generating Fact Checking Summaries for Web Claims", |
| 36 | + author = "Mishra, Rahul and |
| 37 | + Gupta, Dhruv and |
| 38 | + Leippold, Markus", |
| 39 | + editor = "Xu, Wei and |
| 40 | + Ritter, Alan and |
| 41 | + Baldwin, Tim and |
| 42 | + Rahimi, Afshin", |
| 43 | + booktitle = "Proceedings of the Sixth Workshop on Noisy User-generated Text (W-NUT 2020)", |
| 44 | + month = nov, |
| 45 | + year = "2020", |
| 46 | + address = "Online", |
| 47 | + publisher = "Association for Computational Linguistics", |
| 48 | + url = "https://aclanthology.org/2020.wnut-1.12", |
| 49 | + doi = "10.18653/v1/2020.wnut-1.12", |
| 50 | + pages = "81--90", |
| 51 | + abstract = "We present SUMO, a neural attention-based approach that learns to establish correctness of textual claims based on evidence in the form of text documents (e.g., news articles or web documents). SUMO further generates an extractive summary by presenting a diversified set of sentences from the documents that explain its decision on the correctness of the textual claim. Prior approaches to address the problem of fact checking and evidence extraction have relied on simple concatenation of claim and document word embeddings as an input to claim driven attention weight computation. This is done so as to extract salient words and sentences from the documents that help establish the correctness of the claim. However this design of claim-driven attention fails to capture the contextual information in documents properly. We improve on the prior art by using improved claim and title guided hierarchical attention to model effective contextual cues. We show the efficacy of our approach on political, healthcare, and environmental datasets.", |
| 52 | + } |
| 53 | + """ # noqa: E501 |
| 54 | + |
| 55 | + name = "sumosum" |
| 56 | + description = "Text summarization with climate corpus" |
| 57 | + tags = ["summarization", "climate"] |
| 58 | + |
| 59 | + TRAIN_RATIO = 0.2 |
| 60 | + TITLE_KEY = "Title" |
| 61 | + DOCUMENT_KEY = "Doc_text" |
| 62 | + |
| 63 | + def __init__( |
| 64 | + self, |
| 65 | + train_filter_min_length: Optional[int] = None, |
| 66 | + train_filter_max_length: Optional[int] = None, |
| 67 | + test_filter_min_length: Optional[int] = None, |
| 68 | + test_filter_max_length: Optional[int] = None, |
| 69 | + truncate_length: Optional[int] = None, |
| 70 | + ): |
| 71 | + """ |
| 72 | + Initializes the scenario. |
| 73 | + Args: |
| 74 | + train_filter_min_length: Int indicating minimum length for training |
| 75 | + documents. Train examples smaller than |
| 76 | + train_filter_min_length tokens will be filtered out. |
| 77 | + train_filter_max_length: Int indicating maximum length for training |
| 78 | + documents. Train examples larger than |
| 79 | + train_filter_max_length tokens will be filtered out. |
| 80 | + test_filter_min_length: Int indicating minimum length for training |
| 81 | + documents. Test examples smaller than |
| 82 | + test_filter_min_length tokens will be filtered out. |
| 83 | + test_filter_max_length: Int indicating maximum length for training |
| 84 | + documents. Test examples larger than |
| 85 | + test_filter_max_length tokens will be filtered out. |
| 86 | + truncate_length: Int indicating the maximum length in tokens to |
| 87 | + truncate documents. Documents in all splits will be |
| 88 | + truncated to truncate_length tokens. |
| 89 | + NOTE: Whitespace tokenization is used to compute tokens. |
| 90 | + """ |
| 91 | + super().__init__() |
| 92 | + self.train_filter_min_length = train_filter_min_length |
| 93 | + self.train_filter_max_length = train_filter_max_length |
| 94 | + self.test_filter_min_length = test_filter_min_length |
| 95 | + self.test_filter_max_length = test_filter_max_length |
| 96 | + self.truncate_length = truncate_length |
| 97 | + |
| 98 | + @staticmethod |
| 99 | + def _clean_and_truncate(text: str, max_length: Optional[int] = None) -> str: |
| 100 | + text = re.sub(r"\s+", " ", text) |
| 101 | + return " ".join(text.split()[:max_length]) |
| 102 | + |
| 103 | + def _load_dataset(self, output_path: str) -> Dict[str, pd.DataFrame]: |
| 104 | + data_dir = os.path.join(output_path, "data") |
| 105 | + ensure_directory_exists(data_dir) |
| 106 | + |
| 107 | + source_url = "https://github.com/rahulOmishra/SUMO/raw/main/climate_claims_raw.xlsx" |
| 108 | + source_file = os.path.basename(source_url) |
| 109 | + target_path = os.path.join(data_dir, source_file) |
| 110 | + ensure_file_downloaded( |
| 111 | + source_url=source_url, |
| 112 | + target_path=target_path, |
| 113 | + ) |
| 114 | + |
| 115 | + # Column headers: Claim_id(int),Claim,Title,Doc_text,Label(bool) |
| 116 | + target_df = pd.read_excel(target_path, skiprows=1) |
| 117 | + target_df = target_df.dropna(subset=[SUMOSumScenario.TITLE_KEY, SUMOSumScenario.DOCUMENT_KEY]) |
| 118 | + # Remove carriage return _x000D_ in Excel string |
| 119 | + target_df = target_df.replace({r"_x000D_": ""}, regex=True) |
| 120 | + # target_df = target_df.replace({r"_x([0-9a-fA-F]{4})_": ""}, regex=True) |
| 121 | + # Split randomly (works better than split by order) |
| 122 | + train_df = target_df.sample(frac=SUMOSumScenario.TRAIN_RATIO, random_state=0) |
| 123 | + test_df = target_df.drop(train_df.index).sample(frac=1, random_state=0) |
| 124 | + return {TRAIN_SPLIT: train_df, TEST_SPLIT: test_df} |
| 125 | + |
| 126 | + def get_instances(self, output_path: str) -> List[Instance]: |
| 127 | + dataset_dict = self._load_dataset(output_path) |
| 128 | + |
| 129 | + instances: List[Instance] = [] |
| 130 | + |
| 131 | + for split, split_data in dataset_dict.items(): |
| 132 | + for example in split_data.itertuples(): |
| 133 | + document = getattr(example, SUMOSumScenario.DOCUMENT_KEY) |
| 134 | + title = getattr(example, SUMOSumScenario.TITLE_KEY) |
| 135 | + art_len = len(document.split()) |
| 136 | + if split == TEST_SPLIT: |
| 137 | + if self.test_filter_max_length and art_len > self.test_filter_max_length: |
| 138 | + continue |
| 139 | + if self.test_filter_min_length and art_len < self.test_filter_min_length: |
| 140 | + continue |
| 141 | + if split == TRAIN_SPLIT: |
| 142 | + if self.train_filter_max_length and art_len > self.train_filter_max_length: |
| 143 | + continue |
| 144 | + if self.train_filter_min_length and art_len < self.train_filter_min_length: |
| 145 | + continue |
| 146 | + |
| 147 | + document = SUMOSumScenario._clean_and_truncate(document, self.truncate_length) |
| 148 | + title = SUMOSumScenario._clean_and_truncate(title) |
| 149 | + |
| 150 | + instance = Instance( |
| 151 | + input=Input(text=document), |
| 152 | + references=[Reference(output=Output(text=title), tags=[CORRECT_TAG])], |
| 153 | + split=split, |
| 154 | + ) |
| 155 | + instances.append(instance) |
| 156 | + |
| 157 | + return instances |
0 commit comments