dedup _get_model_external_data_paths

Hualiang Xie (from Dev Box) · Hualiang Xie (from Dev Box) · commit fc201487be6b · 2025-03-18T13:55:22.000+08:00
diff --git a/optimum/onnx/utils.py b/optimum/onnx/utils.py
@@ -84,7 +84,7 @@ def _get_model_external_data_paths(model_path: Path) -> List[Path]:
         for tensor in model_tensors
         if tensor.HasField("data_location") and tensor.data_location == onnx.TensorProto.EXTERNAL
     ]
-    return [model_path.parent / tensor_name for tensor_name in model_tensors_ext]
+    return list(dict.fromkeys([model_path.parent / tensor_name for tensor_name in model_tensors_ext]))
 
 
 def check_model_uses_external_data(model: onnx.ModelProto) -> bool:

Original file line number	Diff line number	Diff line change
`@@ -84,7 +84,7 @@ def _get_model_external_data_paths(model_path: Path) -> List[Path]:`
`84`	`84`	`for tensor in model_tensors`
`85`	`85`	`if tensor.HasField("data_location") and tensor.data_location == onnx.TensorProto.EXTERNAL`
`86`	`86`	`]`
`87`		`- return [model_path.parent / tensor_name for tensor_name in model_tensors_ext]`
	`87`	`+ return list(dict.fromkeys([model_path.parent / tensor_name for tensor_name in model_tensors_ext]))`
`88`	`88`
`89`	`89`
`90`	`90`	`def check_model_uses_external_data(model: onnx.ModelProto) -> bool:`