Skip to content

Commit 8fca1d6

Browse files
IlyasMoutawwakilIlyasMoutawwakil
committed
added token argument
1 parent 7db4c7e commit 8fca1d6

12 files changed

+54
-7
lines changed

‎optimum/exporters/openvino/__main__.py

+3
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ def main_export(
5656
force_download: bool = False,
5757
local_files_only: bool = False,
5858
use_auth_token: Optional[Union[bool, str]] = None,
59+
token: Optional[Union[bool, str]] = None,
5960
model_kwargs: Optional[Dict[str, Any]] = None,
6061
custom_export_configs: Optional[Dict[str, "OnnxConfig"]] = None,
6162
fn_get_submodels: Optional[Callable] = None,
@@ -196,6 +197,7 @@ def main_export(
196197
revision=revision,
197198
cache_dir=cache_dir,
198199
use_auth_token=use_auth_token,
200+
token=token,
199201
local_files_only=local_files_only,
200202
force_download=force_download,
201203
trust_remote_code=trust_remote_code,
@@ -268,6 +270,7 @@ class StoreAttr(object):
268270
revision=revision,
269271
cache_dir=cache_dir,
270272
use_auth_token=use_auth_token,
273+
token=token,
271274
local_files_only=local_files_only,
272275
force_download=force_download,
273276
trust_remote_code=trust_remote_code,

‎optimum/intel/generation/modeling.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -353,7 +353,8 @@ def _from_pretrained(
353353
cls,
354354
model_id: Union[str, Path],
355355
config: PretrainedConfig,
356-
use_auth_token: Optional[Union[bool, str, None]] = None,
356+
use_auth_token: Optional[Union[bool, str]] = None,
357+
token: Optional[Union[bool, str]] = None,
357358
revision: Optional[Union[str, None]] = None,
358359
force_download: bool = False,
359360
cache_dir: Optional[str] = None,
@@ -376,6 +377,7 @@ def _from_pretrained(
376377
repo_id=model_id,
377378
filename=file_name,
378379
use_auth_token=use_auth_token,
380+
token=token,
379381
revision=revision,
380382
cache_dir=cache_dir,
381383
force_download=force_download,
@@ -398,6 +400,7 @@ def _from_transformers(
398400
model_id: str,
399401
config: PretrainedConfig,
400402
use_auth_token: Optional[Union[bool, str]] = None,
403+
token: Optional[Union[bool, str]] = None,
401404
revision: Optional[str] = None,
402405
force_download: bool = False,
403406
cache_dir: Optional[str] = None,
@@ -414,6 +417,7 @@ def _from_transformers(
414417
model_kwargs = {
415418
"revision": revision,
416419
"use_auth_token": use_auth_token,
420+
"token": token,
417421
"cache_dir": cache_dir,
418422
"subfolder": subfolder,
419423
"local_files_only": local_files_only,
@@ -436,6 +440,7 @@ def _from_transformers(
436440
config=config,
437441
use_cache=use_cache,
438442
use_auth_token=use_auth_token,
443+
token=token,
439444
revision=revision,
440445
force_download=force_download,
441446
cache_dir=cache_dir,

‎optimum/intel/ipex/modeling_base.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ def _from_transformers(
151151
config: PretrainedConfig,
152152
use_cache: bool = True,
153153
use_auth_token: Optional[Union[bool, str]] = None,
154+
token: Optional[Union[bool, str]] = None,
154155
revision: Optional[str] = None,
155156
force_download: bool = False,
156157
cache_dir: Optional[str] = None,
@@ -166,6 +167,7 @@ def _from_transformers(
166167
model_kwargs = {
167168
"revision": revision,
168169
"use_auth_token": use_auth_token,
170+
"token": token,
169171
"cache_dir": cache_dir,
170172
"subfolder": subfolder,
171173
"local_files_only": local_files_only,
@@ -187,7 +189,8 @@ def _from_pretrained(
187189
cls,
188190
model_id: Union[str, Path],
189191
config: PretrainedConfig,
190-
use_auth_token: Optional[Union[bool, str, None]] = None,
192+
use_auth_token: Optional[Union[bool, str]] = None,
193+
token: Optional[Union[bool, str]] = None,
191194
revision: Optional[Union[str, None]] = None,
192195
force_download: bool = False,
193196
cache_dir: Optional[str] = None,
@@ -211,6 +214,7 @@ def _from_pretrained(
211214
repo_id=model_id,
212215
filename=file_name,
213216
use_auth_token=use_auth_token,
217+
token=token,
214218
revision=revision,
215219
cache_dir=cache_dir,
216220
force_download=force_download,

‎optimum/intel/neural_compressor/modeling_base.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,8 @@ def _from_pretrained(
101101
cls,
102102
model_id: Union[str, Path],
103103
config: PretrainedConfig,
104-
use_auth_token: Optional[Union[bool, str, None]] = None,
104+
use_auth_token: Optional[Union[bool, str]] = None,
105+
token: Optional[Union[bool, str]] = None,
105106
revision: Optional[Union[str, None]] = None,
106107
force_download: bool = False,
107108
cache_dir: Optional[str] = None,
@@ -126,6 +127,7 @@ def _from_pretrained(
126127
filename=file_name,
127128
subfolder=subfolder,
128129
use_auth_token=use_auth_token,
130+
token=token,
129131
revision=revision,
130132
cache_dir=cache_dir,
131133
force_download=force_download,
@@ -149,6 +151,7 @@ def _from_pretrained(
149151
return _BaseQBitsAutoModelClass.from_pretrained(
150152
pretrained_model_name_or_path=model_id,
151153
use_auth_token=use_auth_token,
154+
token=token,
152155
revision=revision,
153156
force_download=force_download,
154157
cache_dir=cache_dir,

‎optimum/intel/neural_compressor/quantization.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -446,7 +446,8 @@ def get_calibration_dataset(
446446
dataset_split: str = "train",
447447
preprocess_function: Optional[Callable] = None,
448448
preprocess_batch: bool = True,
449-
use_auth_token: bool = False,
449+
use_auth_token: Optional[Union[bool, str]] = None,
450+
token: Optional[Union[bool, str]] = None,
450451
) -> Dataset:
451452
"""
452453
Create the calibration `datasets.Dataset` to use for the post-training static quantization calibration step.
@@ -475,6 +476,7 @@ def get_calibration_dataset(
475476
name=dataset_config_name,
476477
split=dataset_split,
477478
use_auth_token=use_auth_token,
479+
token=token,
478480
)
479481

480482
if num_samples is not None:

‎optimum/intel/openvino/loaders.py

+3
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,7 @@ def load_textual_inversion(
258258
proxies = kwargs.pop("proxies", None)
259259
local_files_only = kwargs.pop("local_files_only", HF_HUB_OFFLINE)
260260
use_auth_token = kwargs.pop("use_auth_token", None)
261+
token = kwargs.pop("token", None)
261262
revision = kwargs.pop("revision", None)
262263
subfolder = kwargs.pop("subfolder", None)
263264
weight_name = kwargs.pop("weight_name", None)
@@ -320,6 +321,7 @@ def load_textual_inversion(
320321
proxies=proxies,
321322
local_files_only=local_files_only,
322323
use_auth_token=use_auth_token,
324+
token=token,
323325
revision=revision,
324326
subfolder=subfolder,
325327
user_agent=user_agent,
@@ -341,6 +343,7 @@ def load_textual_inversion(
341343
proxies=proxies,
342344
local_files_only=local_files_only,
343345
use_auth_token=use_auth_token,
346+
token=token,
344347
revision=revision,
345348
subfolder=subfolder,
346349
user_agent=user_agent,

‎optimum/intel/openvino/modeling.py

+4
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,7 @@ def _from_transformers(
421421
model_id: str,
422422
config: PretrainedConfig,
423423
use_auth_token: Optional[Union[bool, str]] = None,
424+
token: Optional[Union[bool, str]] = None,
424425
revision: Optional[str] = None,
425426
force_download: bool = False,
426427
cache_dir: Optional[str] = None,
@@ -450,6 +451,7 @@ def _from_transformers(
450451
revision=revision,
451452
cache_dir=cache_dir,
452453
use_auth_token=use_auth_token,
454+
token=token,
453455
local_files_only=local_files_only,
454456
force_download=force_download,
455457
trust_remote_code=trust_remote_code,
@@ -583,6 +585,7 @@ def from_pretrained(
583585
export: bool = False,
584586
config: Optional["PretrainedConfig"] = None,
585587
use_auth_token: Optional[Union[bool, str]] = None,
588+
token: Optional[Union[bool, str]] = None,
586589
revision: Optional[str] = None,
587590
force_download: bool = False,
588591
cache_dir: Optional[str] = None,
@@ -621,6 +624,7 @@ def from_pretrained(
621624
config=config,
622625
export=export,
623626
use_auth_token=use_auth_token,
627+
token=token,
624628
revision=revision,
625629
force_download=force_download,
626630
cache_dir=cache_dir,

‎optimum/intel/openvino/modeling_base.py

+9-1
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,8 @@ def _from_pretrained(
168168
cls,
169169
model_id: Union[str, Path],
170170
config: PretrainedConfig,
171-
use_auth_token: Optional[Union[bool, str, None]] = None,
171+
use_auth_token: Optional[Union[bool, str]] = None,
172+
token: Optional[Union[bool, str]] = None,
172173
revision: Optional[Union[str, None]] = None,
173174
force_download: bool = False,
174175
cache_dir: Optional[str] = None,
@@ -215,6 +216,7 @@ def _from_pretrained(
215216
model_cache_path = cls._cached_file(
216217
model_path=model_path,
217218
use_auth_token=use_auth_token,
219+
token=token,
218220
revision=revision,
219221
force_download=force_download,
220222
cache_dir=cache_dir,
@@ -260,6 +262,7 @@ def _set_ov_config_parameters(self):
260262
def _cached_file(
261263
model_path: Union[Path, str],
262264
use_auth_token: Optional[Union[bool, str]] = None,
265+
token: Optional[Union[bool, str]] = None,
263266
revision: Optional[str] = None,
264267
force_download: bool = False,
265268
cache_dir: Optional[str] = None,
@@ -283,6 +286,7 @@ def _cached_file(
283286
filename=file_name.as_posix(),
284287
subfolder=subfolder,
285288
use_auth_token=use_auth_token,
289+
token=token,
286290
revision=revision,
287291
cache_dir=cache_dir,
288292
force_download=force_download,
@@ -298,6 +302,7 @@ def _from_transformers(
298302
model_id: str,
299303
config: PretrainedConfig,
300304
use_auth_token: Optional[Union[bool, str]] = None,
305+
token: Optional[Union[bool, str]] = None,
301306
revision: Optional[str] = None,
302307
force_download: bool = False,
303308
cache_dir: Optional[str] = None,
@@ -344,6 +349,7 @@ def _from_transformers(
344349
revision=revision,
345350
cache_dir=cache_dir,
346351
use_auth_token=use_auth_token,
352+
token=token,
347353
local_files_only=local_files_only,
348354
force_download=force_download,
349355
trust_remote_code=trust_remote_code,
@@ -366,6 +372,7 @@ def _to_load(
366372
config: PretrainedConfig,
367373
onnx_config: OnnxConfig,
368374
use_auth_token: Optional[Union[bool, str]] = None,
375+
token: Optional[Union[bool, str]] = None,
369376
revision: Optional[str] = None,
370377
force_download: bool = False,
371378
cache_dir: Optional[str] = None,
@@ -390,6 +397,7 @@ def _to_load(
390397
config=config,
391398
from_onnx=False,
392399
use_auth_token=use_auth_token,
400+
token=token,
393401
revision=revision,
394402
force_download=force_download,
395403
cache_dir=cache_dir,

‎optimum/intel/openvino/modeling_base_seq2seq.py

+4
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ def _from_pretrained(
109109
model_id: Union[str, Path],
110110
config: PretrainedConfig,
111111
use_auth_token: Optional[Union[bool, str]] = None,
112+
token: Optional[Union[bool, str]] = None,
112113
revision: Optional[str] = None,
113114
force_download: bool = False,
114115
cache_dir: Optional[str] = None,
@@ -191,6 +192,7 @@ def _from_pretrained(
191192
repo_id=model_id,
192193
filename=file_name,
193194
use_auth_token=use_auth_token,
195+
token=token,
194196
revision=revision,
195197
cache_dir=cache_dir,
196198
force_download=force_download,
@@ -220,6 +222,7 @@ def _from_transformers(
220222
model_id: str,
221223
config: PretrainedConfig,
222224
use_auth_token: Optional[Union[bool, str]] = None,
225+
token: Optional[Union[bool, str]] = None,
223226
revision: Optional[str] = None,
224227
force_download: bool = False,
225228
cache_dir: Optional[str] = None,
@@ -273,6 +276,7 @@ def _from_transformers(
273276
revision=revision,
274277
cache_dir=cache_dir,
275278
use_auth_token=use_auth_token,
279+
token=token,
276280
local_files_only=local_files_only,
277281
force_download=force_download,
278282
trust_remote_code=trust_remote_code,

‎optimum/intel/openvino/modeling_decoder.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ def _from_transformers(
219219
model_id: str,
220220
config: PretrainedConfig,
221221
use_auth_token: Optional[Union[bool, str]] = None,
222+
token: Optional[Union[bool, str]] = None,
222223
revision: Optional[str] = None,
223224
force_download: bool = False,
224225
cache_dir: Optional[str] = None,
@@ -255,6 +256,7 @@ def _from_transformers(
255256
revision=revision,
256257
cache_dir=cache_dir,
257258
use_auth_token=use_auth_token,
259+
token=token,
258260
local_files_only=local_files_only,
259261
force_download=force_download,
260262
trust_remote_code=trust_remote_code,
@@ -562,7 +564,8 @@ def _from_pretrained(
562564
cls,
563565
model_id: Union[str, Path],
564566
config: PretrainedConfig,
565-
use_auth_token: Optional[Union[bool, str, None]] = None,
567+
use_auth_token: Optional[Union[bool, str]] = None,
568+
token: Optional[Union[bool, str]] = None,
566569
revision: Optional[Union[str, None]] = None,
567570
force_download: bool = False,
568571
cache_dir: Optional[str] = None,
@@ -581,6 +584,7 @@ def _from_pretrained(
581584
model_cache_path = cls._cached_file(
582585
model_path=model_path,
583586
use_auth_token=use_auth_token,
587+
token=token,
584588
revision=revision,
585589
force_download=force_download,
586590
cache_dir=cache_dir,

‎optimum/intel/openvino/modeling_diffusion.py

+5
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,7 @@ def _from_pretrained(
207207
model_id: Union[str, Path],
208208
config: Dict[str, Any],
209209
use_auth_token: Optional[Union[bool, str]] = None,
210+
token: Optional[Union[bool, str]] = None,
210211
revision: Optional[str] = None,
211212
cache_dir: Optional[str] = None,
212213
vae_decoder_file_name: Optional[str] = None,
@@ -260,6 +261,7 @@ def _from_pretrained(
260261
cache_dir=cache_dir,
261262
local_files_only=local_files_only,
262263
use_auth_token=use_auth_token,
264+
token=token,
263265
revision=revision,
264266
allow_patterns=allow_patterns,
265267
ignore_patterns=ignore_patterns,
@@ -398,6 +400,7 @@ def _from_transformers(
398400
model_id: str,
399401
config: Dict[str, Any],
400402
use_auth_token: Optional[Union[bool, str]] = None,
403+
token: Optional[Union[bool, str]] = None,
401404
revision: Optional[str] = None,
402405
force_download: bool = False,
403406
cache_dir: Optional[str] = None,
@@ -428,6 +431,7 @@ def _from_transformers(
428431
revision=revision,
429432
cache_dir=cache_dir,
430433
use_auth_token=use_auth_token,
434+
token=token,
431435
local_files_only=local_files_only,
432436
force_download=force_download,
433437
ov_config=ov_config,
@@ -438,6 +442,7 @@ def _from_transformers(
438442
config=config,
439443
from_onnx=False,
440444
use_auth_token=use_auth_token,
445+
token=token,
441446
revision=revision,
442447
force_download=force_download,
443448
cache_dir=cache_dir,

‎optimum/intel/openvino/quantization.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -540,7 +540,8 @@ def get_calibration_dataset(
540540
dataset_split: str = "train",
541541
preprocess_function: Optional[Callable] = None,
542542
preprocess_batch: bool = True,
543-
use_auth_token: bool = False,
543+
use_auth_token: Optional[Union[bool, str]] = None,
544+
token: Optional[Union[bool, str]] = None,
544545
cache_dir: Optional[str] = None,
545546
) -> datasets.Dataset:
546547
"""
@@ -576,6 +577,7 @@ def get_calibration_dataset(
576577
name=dataset_config_name,
577578
split=dataset_split,
578579
use_auth_token=use_auth_token,
580+
token=token,
579581
cache_dir=cache_dir,
580582
)
581583

0 commit comments

Comments
 (0)