fix bug caused by deleting keys as we iterated through index

closes #1029
dedupeio · Jun 1, 2022 · 1d01808 · oreccb · Jun 8, 2022 · fgregg
1 parent 0ee14ec
commit 1d01808
Showing 1 changed file with 7 additions and 3 deletions.
diff --git a/dedupe/canopy_index.py b/dedupe/canopy_index.py
@@ -21,14 +21,13 @@ def initSearch(self):
         N = len(self.index._docweight)
         threshold = int(max(1000, N * 0.05))
 
+        stop_words = []
         self._wids_dict = {}
 
         bucket = self.index.family.IF.Bucket
         for wid, docs in self.index._wordinfo.items():
             if len(docs) > threshold:
-                word = self.lexicon._words[wid]
-                logger.info("Removing stop word {}".format(word))
-                del self.index._wordinfo[wid]
+                stop_words.append(wid)
                 continue
             if isinstance(docs, dict):
                 docs = bucket(docs)
@@ -37,6 +36,11 @@ def initSearch(self):
             term = self.lexicon._words[wid]
             self._wids_dict[term] = (wid, idf)
 
+        for wid in stop_words:
+            word = self.lexicon._words.pop(wid)
+            logger.info("Removing stop word {}".format(word))
+            del self.index._wordinfo[wid]
+
     def apply(self, query_list, threshold, start=0, count=None):
         _wids_dict = self._wids_dict
         _wordinfo = self.index._wordinfo