[lint] upgrade mypy to latest version

Fixes pytorch#75927.

Had to fix some bugs and add some ignores.

To check if clean:
```
lintrunner --paths-cmd='git grep -Il .' --take MYPY,MYPYSTRICT
```

Pull Request resolved: pytorch#76753

Approved by: https://github.com/malfet

Author: suo

.github/scripts/lint_native_functions.py

@@ -27,9 +27,9 @@ def fn(base: str) -> str:
     contents = f.read()
 
 yaml = ruamel.yaml.YAML()  # type: ignore[attr-defined]
-yaml.preserve_quotes = True
-yaml.width = 1000
-yaml.boolean_representation = ['False', 'True']
+yaml.preserve_quotes = True  # type: ignore[assignment]
+yaml.width = 1000  # type: ignore[assignment]
+yaml.boolean_representation = ['False', 'True']  # type: ignore[attr-defined]
 r = yaml.load(contents)
 
 # Cuz ruamel's author intentionally didn't include conversion to string

.lintrunner.toml

@@ -123,7 +123,14 @@ init_command = [
     '--dry-run={{DRYRUN}}',
     'numpy==1.20',
     'expecttest==0.1.3',
-    'mypy==0.812',
+    'mypy==0.950',
+    'types-requests==2.27.25',
+    'types-six==1.16.15',
+    'types-PyYAML==6.0.7',
+    'types-tabulate==0.8.8',
+    'types-protobuf==3.19.18',
+    'types-pkg-resources==0.1.3',
+    'types-Jinja2==2.11.9',
     'junitparser==2.1.1',
     'rich==10.9.0',
     'pyyaml==6.0',

benchmarks/instruction_counts/core/expand.py

@@ -8,7 +8,7 @@
 import os
 import re
 import textwrap
-from typing import cast, List, Optional, Tuple, TYPE_CHECKING
+from typing import List, Optional, Tuple, TYPE_CHECKING
 import uuid
 
 import torch
@@ -63,15 +63,12 @@ def _generate_torchscript_file(model_src: str, name: str) -> Optional[str]:
 
     # Import magic to actually load our function.
     module_spec = importlib.util.spec_from_file_location(f"torchscript__{name}", module_path)
+    assert module_spec is not None
     module = importlib.util.module_from_spec(module_spec)
     loader = module_spec.loader
     assert loader is not None
 
-    # Module.loader has type Optional[_Loader]. Even when we assert loader is
-    # not None and MyPy narrows it to type _Loader, it will not pass type
-    # checks. So we have to use a cast to tell MyPy that _Loader implements
-    # importlib.abc.Loader.
-    cast(importlib.abc.Loader, loader).exec_module(module)
+    loader.exec_module(module)
 
     # And again, the type checker has no way of knowing that this line is valid.
     jit_model = module.jit_model  # type: ignore[attr-defined]

tools/setup_helpers/cmake.py

@@ -342,7 +342,7 @@ def generate(
         cmake_prefix_path = build_options.get("CMAKE_PREFIX_PATH", None)
         if cmake_prefix_path:
             build_options["CMAKE_PREFIX_PATH"] = (
-                cast(str, py_lib_path) + ";" + cast(str, cmake_prefix_path)
+                py_lib_path + ";" + cast(str, cmake_prefix_path)
             )
         else:
             build_options["CMAKE_PREFIX_PATH"] = py_lib_path

tools/shared/module_loader.py

@@ -7,6 +7,7 @@ def import_module(name: str, path: str) -> ModuleType:
     import importlib.util
 
     spec = importlib.util.spec_from_file_location(name, path)
+    assert spec is not None
     module = importlib.util.module_from_spec(spec)
     cast(Loader, spec.loader).exec_module(module)
     return module

tools/stats/s3_stat_parser.py

@@ -102,7 +102,7 @@ def get_S3_object_from_bucket(bucket_name: str, object: str) -> Any:
 
 def case_status(case: Version1Case) -> Status:
     for k in {"errored", "failed", "skipped"}:
-        if case[k]:  # type: ignore[misc]
+        if case[k]:  # type: ignore[literal-required]
             return cast(Status, k)
     return None
 

tools/test/test_stats.py

@@ -85,7 +85,7 @@ def make_report_v2(
         }
         files[file_name] = {
             "suites": suites,
-            "total_seconds": sum(suite["total_seconds"] for suite in suites.values()),
+            "total_seconds": sum(suite["total_seconds"] for suite in suites.values()),  # type: ignore[type-var]
         }
     return {
         **dummy_meta_meta(),  # type: ignore[misc]

torch/_C/__init__.pyi.in

@@ -54,7 +54,7 @@ class Stream:
 class Size(Tuple[_int, ...]):
     # TODO: __reduce__
 
-    @overload
+    @overload  # type: ignore[override]
     def __getitem__(self: Size, key: _int) -> _int: ...
 
     @overload

torch/_deploy.py

@@ -82,7 +82,7 @@ def persistent_load(saved_id):
         importer = sys_importer
 
     unpickler = PackageUnpickler(importer, io.BytesIO(obj_bytes))
-    unpickler.persistent_load = persistent_load
+    unpickler.persistent_load = persistent_load  # type: ignore[assignment]
     result = _deploy_objects[id] = unpickler.load()
     return result
 

torch/ao/nn/sparse/quantized/linear.py

@@ -169,22 +169,22 @@ def from_float(cls, mod):
         assert hasattr(mod, 'sparse_params'), \
             ('Expecting the Linear to have `sparse_params`. Make sure you have provided arguments '
              'in the `sparsifier.squash_mask(params_to_save=("sparse_block_shape",))` method.')
-        sparse_block_shape = mod.sparse_params.get('sparse_block_shape', None)
+        sparse_block_shape = mod.sparse_params.get('sparse_block_shape', None)  # type: ignore[operator, union-attr]
         assert isinstance(sparse_block_shape, (tuple, list))
         assert len(sparse_block_shape) == 2
         # TODO: Need to add options to qconfig to avoid the calibration.
         # TODO: Add calibration for the sparsity
         assert hasattr(mod, 'qconfig'), 'Input float module must have qconfig defined'
         activation_post_process = mod.activation_post_process
-        weight_post_process = mod.qconfig.weight()
+        weight_post_process = mod.qconfig.weight()  # type: ignore[operator, union-attr]
 
         # Assumption is that the weight is already sparsified by the
         # `sparsifier.convert`
         weight = mod.weight
 
         weight_post_process(weight)
         dtype = weight_post_process.dtype
-        act_scale, act_zp = activation_post_process.calculate_qparams()
+        act_scale, act_zp = activation_post_process.calculate_qparams()  # type: ignore[operator, union-attr]
         assert dtype == torch.qint8, 'Weight observer must have dtype torch.qint8'
         w_sc, w_zp = weight_post_process.calculate_qparams()
         if isinstance(w_zp, torch.Tensor):
@@ -193,15 +193,15 @@ def from_float(cls, mod):
             assert w_zp == 0, 'Weight zero point must map to 0'
         qweight = _quantize_weight(weight.float(), weight_post_process)
 
-        row_block_size = mod.sparse_params['sparse_block_shape'][0]
-        col_block_size = mod.sparse_params['sparse_block_shape'][1]
+        row_block_size = mod.sparse_params['sparse_block_shape'][0]  # type: ignore[index]
+        col_block_size = mod.sparse_params['sparse_block_shape'][1]  # type: ignore[index]
         qlinear = cls(mod.in_features,
                       mod.out_features,
                       row_block_size,
                       col_block_size,
                       dtype=dtype)
         qlinear.set_weight_bias(qweight, mod.bias,
-                                row_block_size, col_block_size)
+                                row_block_size, col_block_size)  # type: ignore[arg-type]
         qlinear.scale = float(act_scale)
         qlinear.zero_point = int(act_zp)
         return qlinear

torch/autograd/functional.py

@@ -686,7 +686,7 @@ def vjp(grad_output):
                             raise RuntimeError(msg)
                         jac_i_el.append(torch.zeros_like(inp_el))
 
-            jacobian += (tuple(torch.stack(jac_i_el, dim=0).view(out.size()
+            jacobian += (tuple(torch.stack(jac_i_el, dim=0).view(out.size()  # type: ignore[operator]
                          + inputs[el_idx].size()) for (el_idx, jac_i_el) in enumerate(jac_i)), )
 
         jacobian = _grad_postprocess(jacobian, create_graph)

torch/distributed/distributed_c10d.py

@@ -1880,7 +1880,7 @@ def broadcast_object_list(object_list, src=0, group=None, device=None):
     if my_rank == src:
         object_tensor = torch.cat(tensor_list)
     else:
-        object_tensor = torch.empty(
+        object_tensor = torch.empty(  # type: ignore[call-overload]
             torch.sum(object_sizes_tensor).item(),  # type: ignore[arg-type]
             dtype=torch.uint8,
         )

torch/distributed/elastic/multiprocessing/api.py

@@ -47,7 +47,7 @@ def __init__(self, msg: str, sigval: signal.Signals) -> None:
         self.sigval = sigval
 
 
-def _terminate_process_handler(signum: int, frame: FrameType) -> None:
+def _terminate_process_handler(signum: int, frame: Optional[FrameType]) -> None:
     """Termination handler that raises exceptions on the main process.
 
     When the process receives death signal(SIGTERM, SIGINT), this termination handler will

torch/fx/experimental/unification/variable.py

@@ -19,7 +19,7 @@ def __new__(cls, *token):
             token = token[0]
 
         obj = object.__new__(cls)
-        obj.token = token
+        obj.token = token  # type: ignore[attr-defined]
         return obj
 
     def __str__(self):

torch/fx/graph.py

@@ -151,7 +151,7 @@ def create_name(self, candidate: str, obj: Optional[Any]) -> str:
             num += 1
             candidate = f'{base}_{num}'
 
-        self._used_names.setdefault(candidate)
+        self._used_names.setdefault(candidate, 0)
         if obj is None:
             self._unassociated_names.add(candidate)
         else:

torch/fx/graph_module.py

@@ -240,7 +240,10 @@ def __init__(self, cls, cls_call):
     def _generate_error_message(frame_summary: traceback.FrameSummary) -> str:
         # auxiliary variables (for readability)
         err_lineno = frame_summary.lineno
-        err_line_len = len(frame_summary.line)
+        assert err_lineno is not None
+        line = frame_summary.line
+        assert line is not None
+        err_line_len = len(line)
         all_src_lines = linecache.getlines(frame_summary.filename)
 
         # constituent substrings of the error message
@@ -260,7 +263,7 @@ def __call__(self, obj, *args, **kwargs):
             if self.cls_call is not None:
                 return self.cls_call(obj, *args, **kwargs)
             else:
-                return super(self.cls, obj).__call__(*args, **kwargs)
+                return super(self.cls, obj).__call__(*args, **kwargs)  # type: ignore[misc]
         except Exception as e:
             assert e.__traceback__
             topmost_framesummary: traceback.FrameSummary = \
@@ -638,7 +641,7 @@ def recompile(self) -> PythonCode:
         cls_call = cls.__call__ if "__call__" in vars(cls) else None
 
         if '_wrapped_call' not in vars(cls):
-            cls._wrapped_call = _WrappedCall(cls, cls_call)
+            cls._wrapped_call = _WrappedCall(cls, cls_call)  # type: ignore[attr-defined]
 
         def call_wrapped(self, *args, **kwargs):
             return self._wrapped_call(self, *args, **kwargs)

torch/hub.py

@@ -72,6 +72,7 @@ def _import_module(name, path):
     import importlib.util
     from importlib.abc import Loader
     spec = importlib.util.spec_from_file_location(name, path)
+    assert spec is not None
     module = importlib.util.module_from_spec(spec)
     assert isinstance(spec.loader, Loader)
     spec.loader.exec_module(module)

torch/jit/_script.py

@@ -13,7 +13,7 @@
 import copy
 import pickle
 import warnings
-from typing import Any, Dict, List, Tuple, Union, Callable
+from typing import Any, Dict, List, Set, Tuple, Union, Callable
 
 
 import torch
@@ -249,7 +249,7 @@ def __init__(cls, name, bases, attrs):  # noqa: B902
         for base in reversed(bases):
             for k, v in getattr(base, "_methods", {}).items():
                 cls._methods[k] = v
-            base_constants = getattr(base, "_constants_set", set())
+            base_constants: Set = getattr(base, "_constants_set", set())
             cls._constants_set = cls._constants_set.union(base_constants)
 
         # find all the script methods of the current class
@@ -417,7 +417,7 @@ def __getattr__(self, attr):
                 return super(RecursiveScriptClass, self).__getattr__(attr)  # type: ignore[misc]
 
             if attr in self._props:
-                return self._props[attr].fget()
+                return self._props[attr].fget()  # type: ignore[call-arg, misc]
 
             return getattr(self._c, attr)
 
@@ -426,7 +426,7 @@ def __setattr__(self, attr, value):
                 return super(RecursiveScriptClass, self).__setattr__(attr, value)
 
             if attr in self._props:
-                return self._props[attr].fset(value)
+                return self._props[attr].fset(value)  # type: ignore[call-arg, misc]
 
             setattr(self._c, attr, value)
 
@@ -1306,7 +1306,7 @@ def forward(self, a) -> MyModule:
         qualified_name = _qualified_name(obj)
         # this is a decorated fn, and we need to the underlying fn and its rcb
         if hasattr(obj, "__script_if_tracing_wrapper"):
-            obj = obj.__original_fn
+            obj = obj.__original_fn  # type: ignore[union-attr]
             _rcb = _jit_internal.createResolutionCallbackFromClosure(obj)
 
         # some functions are explicitly marked as not supported in script mode

torch/jit/frontend.py

@@ -537,9 +537,8 @@ def build_AnnAssign(ctx, stmt):
             raise UnsupportedNodeError(ctx, stmt, reason='without assigned value')
 
         # Disallow type annotations on instance attributes outside of __init__
-        if type(stmt.target) == ast.Attribute and\
-                stmt.target.value.id == "self" and\
-                ctx.funcname != "__init__":
+        if type(stmt.target) == ast.Attribute and \
+                stmt.target.value.id == "self" and ctx.funcname != "__init__":  # type: ignore[attr-defined]
             start = stmt.col_offset
             end = start + len(f"self.{stmt.target.attr}")
             if hasattr(stmt.annotation, 'id'):

torch/nn/modules/module.py

@@ -1943,6 +1943,6 @@ def _replicate_for_data_parallel(self):
         replica._parameters = OrderedDict()
         replica._buffers = replica._buffers.copy()
         replica._modules = replica._modules.copy()
-        replica._is_replica = True
+        replica._is_replica = True  # type: ignore[assignment]
 
         return replica

torch/nn/utils/prune.py

@@ -307,7 +307,7 @@ def add_pruning_method(self, method):
                 + " Found '{}'".format(method._tensor_name)
             )
         # if all checks passed, add to _pruning_methods tuple
-        self._pruning_methods += (method,)
+        self._pruning_methods += (method,)  # type: ignore[operator]
 
     def __len__(self):
         return len(self._pruning_methods)

torch/package/_package_pickler.py

@@ -24,15 +24,15 @@ def __init__(self, importer: Importer, *args, **kwargs):
         # is imported, then the offending library removes its dispatch entries,
         # leaving PackagePickler with a stale dispatch table that may cause
         # unwanted behavior.
-        self.dispatch = _Pickler.dispatch.copy()
-        self.dispatch[FunctionType] = PackagePickler.save_global
+        self.dispatch = _Pickler.dispatch.copy()  # type: ignore[misc]
+        self.dispatch[FunctionType] = PackagePickler.save_global  # type: ignore[assignment]
 
     def save_global(self, obj, name=None):
         # unfortunately the pickler code is factored in a way that
         # forces us to copy/paste this function. The only change is marked
         # CHANGED below.
-        write = self.write
-        memo = self.memo
+        write = self.write  # type: ignore[attr-defined]
+        memo = self.memo  # type: ignore[attr-defined]
 
         # CHANGED: import module from module environment instead of __import__
         try:
@@ -44,7 +44,7 @@ def save_global(self, obj, name=None):
         _, parent = _getattribute(module, name)
         # END CHANGED
 
-        if self.proto >= 2:
+        if self.proto >= 2:  # type: ignore[attr-defined]
             code = _extension_registry.get((module_name, name))
             if code:
                 assert code > 0
@@ -59,13 +59,13 @@ def save_global(self, obj, name=None):
         if parent is module:
             name = lastname
         # Non-ASCII identifiers are supported only with protocols >= 3.
-        if self.proto >= 4:
-            self.save(module_name)
-            self.save(name)
+        if self.proto >= 4:  # type: ignore[attr-defined]
+            self.save(module_name)  # type: ignore[attr-defined]
+            self.save(name)  # type: ignore[attr-defined]
             write(STACK_GLOBAL)
         elif parent is not module:
-            self.save_reduce(getattr, (parent, lastname))
-        elif self.proto >= 3:
+            self.save_reduce(getattr, (parent, lastname))  # type: ignore[attr-defined]
+        elif self.proto >= 3:  # type: ignore[attr-defined]
             write(
                 GLOBAL
                 + bytes(module_name, "utf-8")
@@ -74,7 +74,7 @@ def save_global(self, obj, name=None):
                 + b"\n"
             )
         else:
-            if self.fix_imports:
+            if self.fix_imports:  # type: ignore[attr-defined]
                 r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
                 r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
                 if (module_name, name) in r_name_mapping:
@@ -92,10 +92,10 @@ def save_global(self, obj, name=None):
             except UnicodeEncodeError:
                 raise PicklingError(
                     "can't pickle global identifier '%s.%s' using "
-                    "pickle protocol %i" % (module, name, self.proto)
+                    "pickle protocol %i" % (module, name, self.proto)  # type: ignore[attr-defined]
                 ) from None
 
-        self.memoize(obj)
+        self.memoize(obj)  # type: ignore[attr-defined]
 
 
 def create_pickler(data_buf, importer, protocol=4):

torch/package/_package_unpickler.py

@@ -17,7 +17,7 @@ def __init__(self, importer: Importer, *args, **kwargs):
 
     def find_class(self, module, name):
         # Subclasses may override this.
-        if self.proto < 3 and self.fix_imports:
+        if self.proto < 3 and self.fix_imports:  # type: ignore[attr-defined]
             if (module, name) in _compat_pickle.NAME_MAPPING:
                 module, name = _compat_pickle.NAME_MAPPING[(module, name)]
             elif module in _compat_pickle.IMPORT_MAPPING:

torch/package/package_importer.py

@@ -237,7 +237,7 @@ def persistent_load(saved_id):
         # Load the data (which may in turn use `persistent_load` to load tensors)
         data_file = io.BytesIO(self.zip_reader.get_record(pickle_file))
         unpickler = self.Unpickler(data_file)
-        unpickler.persistent_load = persistent_load
+        unpickler.persistent_load = persistent_load  # type: ignore[assignment]
 
         @contextmanager
         def set_deserialization_context():