Major fixes and new features

venv/lib/python3.12/site-packages/mypy/server/astdiff.py

@@ -0,0 +1,516 @@
+"""Utilities for comparing two versions of a module symbol table.
+
+The goal is to find which AST nodes have externally visible changes, so
+that we can fire triggers and re-process other parts of the program
+that are stale because of the changes.
+
+Only look at detail at definitions at the current module -- don't
+recurse into other modules.
+
+A summary of the module contents:
+
+* snapshot_symbol_table(...) creates an opaque snapshot description of a
+  module/class symbol table (recursing into nested class symbol tables).
+
+* compare_symbol_table_snapshots(...) compares two snapshots for the same
+  module id and returns fully qualified names of differences (which act as
+  triggers).
+
+To compare two versions of a module symbol table, take snapshots of both
+versions and compare the snapshots. The use of snapshots makes it easy to
+compare two versions of the *same* symbol table that is being mutated.
+
+Summary of how this works for certain kinds of differences:
+
+* If a symbol table node is deleted or added (only present in old/new version
+  of the symbol table), it is considered different, of course.
+
+* If a symbol table node refers to a different sort of thing in the new version,
+  it is considered different (for example, if a class is replaced with a
+  function).
+
+* If the signature of a function has changed, it is considered different.
+
+* If the type of a variable changes, it is considered different.
+
+* If the MRO of a class changes, or a non-generic class is turned into a
+  generic class, the class is considered different (there are other such "big"
+  differences that cause a class to be considered changed). However, just changes
+  to attributes or methods don't generally constitute a difference at the
+  class level -- these are handled at attribute level (say, 'mod.Cls.method'
+  is different rather than 'mod.Cls' being different).
+
+* If an imported name targets a different name (say, 'from x import y' is
+  replaced with 'from z import y'), the name in the module is considered
+  different. If the target of an import continues to have the same name,
+  but it's specifics change, this doesn't mean that the imported name is
+  treated as changed. Say, there is 'from x import y' in 'm', and the
+  type of 'x.y' has changed. This doesn't mean that that 'm.y' is considered
+  changed. Instead, processing the difference in 'm' will be handled through
+  fine-grained dependencies.
+"""
+
+from __future__ import annotations
+
+from typing import Sequence, Tuple, Union
+from typing_extensions import TypeAlias as _TypeAlias
+
+from mypy.expandtype import expand_type
+from mypy.nodes import (
+    UNBOUND_IMPORTED,
+    Decorator,
+    FuncBase,
+    FuncDef,
+    FuncItem,
+    MypyFile,
+    OverloadedFuncDef,
+    ParamSpecExpr,
+    SymbolNode,
+    SymbolTable,
+    TypeAlias,
+    TypeInfo,
+    TypeVarExpr,
+    TypeVarTupleExpr,
+    Var,
+)
+from mypy.semanal_shared import find_dataclass_transform_spec
+from mypy.types import (
+    AnyType,
+    CallableType,
+    DeletedType,
+    ErasedType,
+    Instance,
+    LiteralType,
+    NoneType,
+    Overloaded,
+    Parameters,
+    ParamSpecType,
+    PartialType,
+    TupleType,
+    Type,
+    TypeAliasType,
+    TypedDictType,
+    TypeType,
+    TypeVarId,
+    TypeVarLikeType,
+    TypeVarTupleType,
+    TypeVarType,
+    TypeVisitor,
+    UnboundType,
+    UninhabitedType,
+    UnionType,
+    UnpackType,
+)
+from mypy.util import get_prefix
+
+# Snapshot representation of a symbol table node or type. The representation is
+# opaque -- the only supported operations are comparing for equality and
+# hashing (latter for type snapshots only). Snapshots can contain primitive
+# objects, nested tuples, lists and dictionaries and primitive objects (type
+# snapshots are immutable).
+#
+# For example, the snapshot of the 'int' type is ('Instance', 'builtins.int', ()).
+
+# Type snapshots are strict, they must be hashable and ordered (e.g. for Unions).
+Primitive: _TypeAlias = Union[str, float, int, bool]  # float is for Literal[3.14] support.
+SnapshotItem: _TypeAlias = Tuple[Union[Primitive, "SnapshotItem"], ...]
+
+# Symbol snapshots can be more lenient.
+SymbolSnapshot: _TypeAlias = Tuple[object, ...]
+
+
+def compare_symbol_table_snapshots(
+    name_prefix: str, snapshot1: dict[str, SymbolSnapshot], snapshot2: dict[str, SymbolSnapshot]
+) -> set[str]:
+    """Return names that are different in two snapshots of a symbol table.
+
+    Only shallow (intra-module) differences are considered. References to things defined
+    outside the module are compared based on the name of the target only.
+
+    Recurse into class symbol tables (if the class is defined in the target module).
+
+    Return a set of fully-qualified names (e.g., 'mod.func' or 'mod.Class.method').
+    """
+    # Find names only defined only in one version.
+    names1 = {f"{name_prefix}.{name}" for name in snapshot1}
+    names2 = {f"{name_prefix}.{name}" for name in snapshot2}
+    triggers = names1 ^ names2
+
+    # Look for names defined in both versions that are different.
+    for name in set(snapshot1.keys()) & set(snapshot2.keys()):
+        item1 = snapshot1[name]
+        item2 = snapshot2[name]
+        kind1 = item1[0]
+        kind2 = item2[0]
+        item_name = f"{name_prefix}.{name}"
+        if kind1 != kind2:
+            # Different kind of node in two snapshots -> trivially different.
+            triggers.add(item_name)
+        elif kind1 == "TypeInfo":
+            if item1[:-1] != item2[:-1]:
+                # Record major difference (outside class symbol tables).
+                triggers.add(item_name)
+            # Look for differences in nested class symbol table entries.
+            assert isinstance(item1[-1], dict)
+            assert isinstance(item2[-1], dict)
+            triggers |= compare_symbol_table_snapshots(item_name, item1[-1], item2[-1])
+        else:
+            # Shallow node (no interesting internal structure). Just use equality.
+            if snapshot1[name] != snapshot2[name]:
+                triggers.add(item_name)
+
+    return triggers
+
+
+def snapshot_symbol_table(name_prefix: str, table: SymbolTable) -> dict[str, SymbolSnapshot]:
+    """Create a snapshot description that represents the state of a symbol table.
+
+    The snapshot has a representation based on nested tuples and dicts
+    that makes it easy and fast to find differences.
+
+    Only "shallow" state is included in the snapshot -- references to
+    things defined in other modules are represented just by the names of
+    the targets.
+    """
+    result: dict[str, SymbolSnapshot] = {}
+    for name, symbol in table.items():
+        node = symbol.node
+        # TODO: cross_ref?
+        fullname = node.fullname if node else None
+        common = (fullname, symbol.kind, symbol.module_public)
+        if isinstance(node, MypyFile):
+            # This is a cross-reference to another module.
+            # If the reference is busted because the other module is missing,
+            # the node will be a "stale_info" TypeInfo produced by fixup,
+            # but that doesn't really matter to us here.
+            result[name] = ("Moduleref", common)
+        elif isinstance(node, TypeVarExpr):
+            result[name] = (
+                "TypeVar",
+                node.variance,
+                [snapshot_type(value) for value in node.values],
+                snapshot_type(node.upper_bound),
+                snapshot_type(node.default),
+            )
+        elif isinstance(node, TypeAlias):
+            result[name] = (
+                "TypeAlias",
+                snapshot_types(node.alias_tvars),
+                node.normalized,
+                node.no_args,
+                snapshot_optional_type(node.target),
+            )
+        elif isinstance(node, ParamSpecExpr):
+            result[name] = (
+                "ParamSpec",
+                node.variance,
+                snapshot_type(node.upper_bound),
+                snapshot_type(node.default),
+            )
+        elif isinstance(node, TypeVarTupleExpr):
+            result[name] = (
+                "TypeVarTuple",
+                node.variance,
+                snapshot_type(node.upper_bound),
+                snapshot_type(node.default),
+            )
+        else:
+            assert symbol.kind != UNBOUND_IMPORTED
+            if node and get_prefix(node.fullname) != name_prefix:
+                # This is a cross-reference to a node defined in another module.
+                result[name] = ("CrossRef", common)
+            else:
+                result[name] = snapshot_definition(node, common)
+    return result
+
+
+def snapshot_definition(node: SymbolNode | None, common: SymbolSnapshot) -> SymbolSnapshot:
+    """Create a snapshot description of a symbol table node.
+
+    The representation is nested tuples and dicts. Only externally
+    visible attributes are included.
+    """
+    if isinstance(node, FuncBase):
+        # TODO: info
+        if node.type:
+            signature = snapshot_type(node.type)
+        else:
+            signature = snapshot_untyped_signature(node)
+        impl: FuncDef | None = None
+        if isinstance(node, FuncDef):
+            impl = node
+        elif isinstance(node, OverloadedFuncDef) and node.impl:
+            impl = node.impl.func if isinstance(node.impl, Decorator) else node.impl
+        is_trivial_body = impl.is_trivial_body if impl else False
+        dataclass_transform_spec = find_dataclass_transform_spec(node)
+        return (
+            "Func",
+            common,
+            node.is_property,
+            node.is_final,
+            node.is_class,
+            node.is_static,
+            signature,
+            is_trivial_body,
+            dataclass_transform_spec.serialize() if dataclass_transform_spec is not None else None,
+        )
+    elif isinstance(node, Var):
+        return ("Var", common, snapshot_optional_type(node.type), node.is_final)
+    elif isinstance(node, Decorator):
+        # Note that decorated methods are represented by Decorator instances in
+        # a symbol table since we need to preserve information about the
+        # decorated function (whether it's a class function, for
+        # example). Top-level decorated functions, however, are represented by
+        # the corresponding Var node, since that happens to provide enough
+        # context.
+        return (
+            "Decorator",
+            node.is_overload,
+            snapshot_optional_type(node.var.type),
+            snapshot_definition(node.func, common),
+        )
+    elif isinstance(node, TypeInfo):
+        dataclass_transform_spec = node.dataclass_transform_spec
+        if dataclass_transform_spec is None:
+            dataclass_transform_spec = find_dataclass_transform_spec(node)
+
+        attrs = (
+            node.is_abstract,
+            node.is_enum,
+            node.is_protocol,
+            node.fallback_to_any,
+            node.meta_fallback_to_any,
+            node.is_named_tuple,
+            node.is_newtype,
+            # We need this to e.g. trigger metaclass calculation in subclasses.
+            snapshot_optional_type(node.metaclass_type),
+            snapshot_optional_type(node.tuple_type),
+            snapshot_optional_type(node.typeddict_type),
+            [base.fullname for base in node.mro],
+            # Note that the structure of type variables is a part of the external interface,
+            # since creating instances might fail, for example:
+            #     T = TypeVar('T', bound=int)
+            #     class C(Generic[T]):
+            #         ...
+            #     x: C[str] <- this is invalid, and needs to be re-checked if `T` changes.
+            # An alternative would be to create both deps: <...> -> C, and <...> -> <C>,
+            # but this currently seems a bit ad hoc.
+            tuple(snapshot_type(tdef) for tdef in node.defn.type_vars),
+            [snapshot_type(base) for base in node.bases],
+            [snapshot_type(p) for p in node._promote],
+            dataclass_transform_spec.serialize() if dataclass_transform_spec is not None else None,
+        )
+        prefix = node.fullname
+        symbol_table = snapshot_symbol_table(prefix, node.names)
+        # Special dependency for abstract attribute handling.
+        symbol_table["(abstract)"] = ("Abstract", tuple(sorted(node.abstract_attributes)))
+        return ("TypeInfo", common, attrs, symbol_table)
+    else:
+        # Other node types are handled elsewhere.
+        assert False, type(node)
+
+
+def snapshot_type(typ: Type) -> SnapshotItem:
+    """Create a snapshot representation of a type using nested tuples."""
+    return typ.accept(SnapshotTypeVisitor())
+
+
+def snapshot_optional_type(typ: Type | None) -> SnapshotItem:
+    if typ:
+        return snapshot_type(typ)
+    else:
+        return ("<not set>",)
+
+
+def snapshot_types(types: Sequence[Type]) -> SnapshotItem:
+    return tuple(snapshot_type(item) for item in types)
+
+
+def snapshot_simple_type(typ: Type) -> SnapshotItem:
+    return (type(typ).__name__,)
+
+
+def encode_optional_str(s: str | None) -> str:
+    if s is None:
+        return "<None>"
+    else:
+        return s
+
+
+class SnapshotTypeVisitor(TypeVisitor[SnapshotItem]):
+    """Creates a read-only, self-contained snapshot of a type object.
+
+    Properties of a snapshot:
+
+    - Contains (nested) tuples and other immutable primitive objects only.
+    - References to AST nodes are replaced with full names of targets.
+    - Has no references to mutable or non-primitive objects.
+    - Two snapshots represent the same object if and only if they are
+      equal.
+    - Results must be sortable. It's important that tuples have
+      consistent types and can't arbitrarily mix str and None values,
+      for example, since they can't be compared.
+    """
+
+    def visit_unbound_type(self, typ: UnboundType) -> SnapshotItem:
+        return (
+            "UnboundType",
+            typ.name,
+            typ.optional,
+            typ.empty_tuple_index,
+            snapshot_types(typ.args),
+        )
+
+    def visit_any(self, typ: AnyType) -> SnapshotItem:
+        return snapshot_simple_type(typ)
+
+    def visit_none_type(self, typ: NoneType) -> SnapshotItem:
+        return snapshot_simple_type(typ)
+
+    def visit_uninhabited_type(self, typ: UninhabitedType) -> SnapshotItem:
+        return snapshot_simple_type(typ)
+
+    def visit_erased_type(self, typ: ErasedType) -> SnapshotItem:
+        return snapshot_simple_type(typ)
+
+    def visit_deleted_type(self, typ: DeletedType) -> SnapshotItem:
+        return snapshot_simple_type(typ)
+
+    def visit_instance(self, typ: Instance) -> SnapshotItem:
+        return (
+            "Instance",
+            encode_optional_str(typ.type.fullname),
+            snapshot_types(typ.args),
+            ("None",) if typ.last_known_value is None else snapshot_type(typ.last_known_value),
+        )
+
+    def visit_type_var(self, typ: TypeVarType) -> SnapshotItem:
+        return (
+            "TypeVar",
+            typ.name,
+            typ.fullname,
+            typ.id.raw_id,
+            typ.id.meta_level,
+            snapshot_types(typ.values),
+            snapshot_type(typ.upper_bound),
+            snapshot_type(typ.default),
+            typ.variance,
+        )
+
+    def visit_param_spec(self, typ: ParamSpecType) -> SnapshotItem:
+        return (
+            "ParamSpec",
+            typ.id.raw_id,
+            typ.id.meta_level,
+            typ.flavor,
+            snapshot_type(typ.upper_bound),
+            snapshot_type(typ.default),
+        )
+
+    def visit_type_var_tuple(self, typ: TypeVarTupleType) -> SnapshotItem:
+        return (
+            "TypeVarTupleType",
+            typ.id.raw_id,
+            typ.id.meta_level,
+            snapshot_type(typ.upper_bound),
+            snapshot_type(typ.default),
+        )
+
+    def visit_unpack_type(self, typ: UnpackType) -> SnapshotItem:
+        return ("UnpackType", snapshot_type(typ.type))
+
+    def visit_parameters(self, typ: Parameters) -> SnapshotItem:
+        return (
+            "Parameters",
+            snapshot_types(typ.arg_types),
+            tuple(encode_optional_str(name) for name in typ.arg_names),
+            tuple(k.value for k in typ.arg_kinds),
+        )
+
+    def visit_callable_type(self, typ: CallableType) -> SnapshotItem:
+        if typ.is_generic():
+            typ = self.normalize_callable_variables(typ)
+        return (
+            "CallableType",
+            snapshot_types(typ.arg_types),
+            snapshot_type(typ.ret_type),
+            tuple(encode_optional_str(name) for name in typ.arg_names),
+            tuple(k.value for k in typ.arg_kinds),
+            typ.is_type_obj(),
+            typ.is_ellipsis_args,
+            snapshot_types(typ.variables),
+        )
+
+    def normalize_callable_variables(self, typ: CallableType) -> CallableType:
+        """Normalize all type variable ids to run from -1 to -len(variables)."""
+        tvs = []
+        tvmap: dict[TypeVarId, Type] = {}
+        for i, v in enumerate(typ.variables):
+            tid = TypeVarId(-1 - i)
+            if isinstance(v, TypeVarType):
+                tv: TypeVarLikeType = v.copy_modified(id=tid)
+            elif isinstance(v, TypeVarTupleType):
+                tv = v.copy_modified(id=tid)
+            else:
+                assert isinstance(v, ParamSpecType)
+                tv = v.copy_modified(id=tid)
+            tvs.append(tv)
+            tvmap[v.id] = tv
+        return expand_type(typ, tvmap).copy_modified(variables=tvs)
+
+    def visit_tuple_type(self, typ: TupleType) -> SnapshotItem:
+        return ("TupleType", snapshot_types(typ.items))
+
+    def visit_typeddict_type(self, typ: TypedDictType) -> SnapshotItem:
+        items = tuple((key, snapshot_type(item_type)) for key, item_type in typ.items.items())
+        required = tuple(sorted(typ.required_keys))
+        return ("TypedDictType", items, required)
+
+    def visit_literal_type(self, typ: LiteralType) -> SnapshotItem:
+        return ("LiteralType", snapshot_type(typ.fallback), typ.value)
+
+    def visit_union_type(self, typ: UnionType) -> SnapshotItem:
+        # Sort and remove duplicates so that we can use equality to test for
+        # equivalent union type snapshots.
+        items = {snapshot_type(item) for item in typ.items}
+        normalized = tuple(sorted(items))
+        return ("UnionType", normalized)
+
+    def visit_overloaded(self, typ: Overloaded) -> SnapshotItem:
+        return ("Overloaded", snapshot_types(typ.items))
+
+    def visit_partial_type(self, typ: PartialType) -> SnapshotItem:
+        # A partial type is not fully defined, so the result is indeterminate. We shouldn't
+        # get here.
+        raise RuntimeError
+
+    def visit_type_type(self, typ: TypeType) -> SnapshotItem:
+        return ("TypeType", snapshot_type(typ.item))
+
+    def visit_type_alias_type(self, typ: TypeAliasType) -> SnapshotItem:
+        assert typ.alias is not None
+        return ("TypeAliasType", typ.alias.fullname, snapshot_types(typ.args))
+
+
+def snapshot_untyped_signature(func: OverloadedFuncDef | FuncItem) -> SymbolSnapshot:
+    """Create a snapshot of the signature of a function that has no explicit signature.
+
+    If the arguments to a function without signature change, it must be
+    considered as different. We have this special casing since we don't store
+    the implicit signature anywhere, and we'd rather not construct new
+    Callable objects in this module (the idea is to only read properties of
+    the AST here).
+    """
+    if isinstance(func, FuncItem):
+        return (tuple(func.arg_names), tuple(func.arg_kinds))
+    else:
+        result: list[SymbolSnapshot] = []
+        for item in func.items:
+            if isinstance(item, Decorator):
+                if item.var.type:
+                    result.append(snapshot_type(item.var.type))
+                else:
+                    result.append(("DecoratorWithoutType",))
+            else:
+                result.append(snapshot_untyped_signature(item))
+        return tuple(result)