Call isDeeplyNestedType only when something changed

src/compiler/checker.ts

@@ -18213,7 +18213,7 @@ namespace ts {
                 if (source.flags & TypeFlags.UnionOrIntersection || target.flags & TypeFlags.UnionOrIntersection) {
                     result = getConstituentCount(source) * getConstituentCount(target) >= 4 ?
                         recursiveTypeRelatedTo(source, target, reportErrors, intersectionState | IntersectionState.UnionIntersectionCheck, recursionFlags) :
-                        structuredTypeRelatedTo(source, target, reportErrors, intersectionState | IntersectionState.UnionIntersectionCheck);
+                        recursiveTypeRelatedToUncached(source, target, reportErrors, intersectionState | IntersectionState.UnionIntersectionCheck, recursionFlags);
                 }
                 if (!result && !(source.flags & TypeFlags.Union) && (source.flags & (TypeFlags.StructuredOrInstantiable) || target.flags & TypeFlags.StructuredOrInstantiable)) {
                     if (result = recursiveTypeRelatedTo(source, target, reportErrors, intersectionState, recursionFlags)) {
@@ -18626,9 +18626,6 @@ namespace ts {
             // equal and infinitely expanding. Fourth, if we have reached a depth of 100 nested comparisons, assume we have runaway recursion
             // and issue an error. Otherwise, actually compare the structure of the two types.
             function recursiveTypeRelatedTo(source: Type, target: Type, reportErrors: boolean, intersectionState: IntersectionState, recursionFlags: RecursionFlags): Ternary {
-                if (overflow) {
-                    return Ternary.False;
-                }
                 const id = getRelationKey(source, target, intersectionState | (inPropertyCheck ? IntersectionState.InPropertyCheck : 0), relation);
                 const entry = relation.get(id);
                 if (entry !== undefined) {
@@ -18652,8 +18649,6 @@ namespace ts {
                 }
                 if (!maybeKeys) {
                     maybeKeys = [];
-                    sourceStack = [];
-                    targetStack = [];
                 }
                 else {
                     // generate a key where all type parameter id positions are replaced with unconstrained type parameter ids
@@ -18669,25 +18664,10 @@ namespace ts {
                             return Ternary.Maybe;
                         }
                     }
-                    if (sourceDepth === 100 || targetDepth === 100) {
-                        overflow = true;
-                        return Ternary.False;
-                    }
                 }
                 const maybeStart = maybeCount;
                 maybeKeys[maybeCount] = id;
                 maybeCount++;
-                if (recursionFlags & RecursionFlags.Source) {
-                    sourceStack[sourceDepth] = source;
-                    sourceDepth++;
-                }
-                if (recursionFlags & RecursionFlags.Target) {
-                    targetStack[targetDepth] = target;
-                    targetDepth++;
-                }
-                const saveExpandingFlags = expandingFlags;
-                if (!(expandingFlags & ExpandingFlags.Source) && isDeeplyNestedType(source, sourceStack, sourceDepth)) expandingFlags |= ExpandingFlags.Source;
-                if (!(expandingFlags & ExpandingFlags.Target) && isDeeplyNestedType(target, targetStack, targetDepth)) expandingFlags |= ExpandingFlags.Target;
                 let originalHandler: typeof outofbandVarianceMarkerHandler;
                 let propagatingVarianceFlags: RelationComparisonResult = 0;
                 if (outofbandVarianceMarkerHandler) {
@@ -18697,29 +18677,10 @@ namespace ts {
                         return originalHandler!(onlyUnreliable);
                     };
                 }
-
-                if (expandingFlags === ExpandingFlags.Both) {
-                    tracing?.instant(tracing.Phase.CheckTypes, "recursiveTypeRelatedTo_DepthLimit", {
-                        sourceId: source.id,
-                        sourceIdStack: sourceStack.map(t => t.id),
-                        targetId: target.id,
-                        targetIdStack: targetStack.map(t => t.id),
-                        depth: sourceDepth,
-                        targetDepth
-                    });
-                }
-
-                const result = expandingFlags !== ExpandingFlags.Both ? structuredTypeRelatedTo(source, target, reportErrors, intersectionState) : Ternary.Maybe;
+                const result = recursiveTypeRelatedToUncached(source, target, reportErrors, intersectionState, recursionFlags);
                 if (outofbandVarianceMarkerHandler) {
                     outofbandVarianceMarkerHandler = originalHandler;
                 }
-                expandingFlags = saveExpandingFlags;
-                if (recursionFlags & RecursionFlags.Source) {
-                    sourceDepth--;
-                }
-                if (recursionFlags & RecursionFlags.Target) {
-                    targetDepth--;
-                }
                 if (result) {
                     if (result === Ternary.True || (sourceDepth === 0 && targetDepth === 0)) {
                         if (result === Ternary.True || result === Ternary.Maybe) {
@@ -18741,14 +18702,52 @@ namespace ts {
                 return result;
             }
 
-            function structuredTypeRelatedTo(source: Type, target: Type, reportErrors: boolean, intersectionState: IntersectionState): Ternary {
-                tracing?.push(tracing.Phase.CheckTypes, "structuredTypeRelatedTo", { sourceId: source.id, targetId: target.id });
-                const result = structuredTypeRelatedToWorker(source, target, reportErrors, intersectionState);
-                tracing?.pop();
+            function recursiveTypeRelatedToUncached(source: Type, target: Type, reportErrors: boolean, intersectionState: IntersectionState, recursionFlags: RecursionFlags): Ternary {
+                if (sourceDepth >= 100 || targetDepth >= 100) {
+                    overflow = true;
+                }
+                if (overflow) {
+                    return Ternary.False;
+                }
+                let result;
+                const saveExpandingFlags = expandingFlags;
+                if (recursionFlags & RecursionFlags.Source) {
+                    (sourceStack || (sourceStack = []))[sourceDepth] = source;
+                    sourceDepth++;
+                    if (!(expandingFlags & ExpandingFlags.Source) && isDeeplyNestedType(source, sourceStack, sourceDepth)) expandingFlags |= ExpandingFlags.Source;
+                }
+                if (recursionFlags & RecursionFlags.Target) {
+                    (targetStack || (targetStack = []))[targetDepth] = target;
+                    targetDepth++;
+                    if (!(expandingFlags & ExpandingFlags.Target) && isDeeplyNestedType(target, targetStack, targetDepth)) expandingFlags |= ExpandingFlags.Target;
+                }
+                if (expandingFlags !== ExpandingFlags.Both) {
+                    tracing?.push(tracing.Phase.CheckTypes, "structuredTypeRelatedTo", { sourceId: source.id, targetId: target.id });
+                    result = structuredTypeRelatedTo(source, target, reportErrors, intersectionState);
+                    tracing?.pop();
+                }
+                else {
+                    tracing?.instant(tracing.Phase.CheckTypes, "recursiveTypeRelatedTo_DepthLimit", {
+                        sourceId: source.id,
+                        sourceIdStack: sourceStack.map(t => t.id),
+                        targetId: target.id,
+                        targetIdStack: targetStack.map(t => t.id),
+                        depth: sourceDepth,
+                        targetDepth
+                    });
+                    result = Ternary.Maybe;
+                }
+                if (recursionFlags & RecursionFlags.Source) {
+                    sourceDepth--;
+                }
+                if (recursionFlags & RecursionFlags.Target) {
+                    targetDepth--;
+                }
+                expandingFlags = saveExpandingFlags;
                 return result;
             }
 
-            function structuredTypeRelatedToWorker(source: Type, target: Type, reportErrors: boolean, intersectionState: IntersectionState): Ternary {
+            function structuredTypeRelatedTo(source: Type, target: Type, reportErrors: boolean, intersectionState: IntersectionState): Ternary {
                 if (intersectionState & IntersectionState.PropertyCheck) {
                     return propertiesRelatedTo(source, target, reportErrors, /*excludedProperties*/ undefined, IntersectionState.None);
                 }

tests/baselines/reference/deepComparisons.errors.txt

@@ -46,4 +46,23 @@ tests/cases/compiler/deepComparisons.ts(4,9): error TS2322: Type 'T[K1][K2]' is
     
     function f3<U>() {
         let x: Foo1<U> = 0 as any as Bar<U>;  // No error!
-    }
+    }
+    
+    // Repro from #46500
+    
+    type F<T> = {} & (
+        T extends [any, ...any[]]
+            ? { [K in keyof T]?: F<T[K]> }
+            : T extends any[]
+                ? F<T[number]>[]
+                : T extends { [K: string]: any }
+                    ? { [K in keyof T]?: F<T[K]> }
+                    : { x: string }
+    );
+    
+    declare function f<T = any>(): F<T>;
+    
+    function g() {
+        return f() as F<any>;
+    }
+    

tests/baselines/reference/deepComparisons.js

@@ -17,7 +17,26 @@ type Foo2<T> = { x: Foo1<T> };
 
 function f3<U>() {
     let x: Foo1<U> = 0 as any as Bar<U>;  // No error!
-}
+}
+
+// Repro from #46500
+
+type F<T> = {} & (
+    T extends [any, ...any[]]
+        ? { [K in keyof T]?: F<T[K]> }
+        : T extends any[]
+            ? F<T[number]>[]
+            : T extends { [K: string]: any }
+                ? { [K in keyof T]?: F<T[K]> }
+                : { x: string }
+);
+
+declare function f<T = any>(): F<T>;
+
+function g() {
+    return f() as F<any>;
+}
+
 
 //// [deepComparisons.js]
 function f1() {
@@ -31,3 +50,6 @@ function f2() {
 function f3() {
     var x = 0; // No error!
 }
+function g() {
+    return f();
+}

tests/baselines/reference/deepComparisons.symbols

@@ -84,3 +84,57 @@ function f3<U>() {
 >Bar : Symbol(Bar, Decl(deepComparisons.ts, 6, 28))
 >U : Symbol(U, Decl(deepComparisons.ts, 16, 12))
 }
+
+// Repro from #46500
+
+type F<T> = {} & (
+>F : Symbol(F, Decl(deepComparisons.ts, 18, 1))
+>T : Symbol(T, Decl(deepComparisons.ts, 22, 7))
+
+    T extends [any, ...any[]]
+>T : Symbol(T, Decl(deepComparisons.ts, 22, 7))
+
+        ? { [K in keyof T]?: F<T[K]> }
+>K : Symbol(K, Decl(deepComparisons.ts, 24, 13))
+>T : Symbol(T, Decl(deepComparisons.ts, 22, 7))
+>F : Symbol(F, Decl(deepComparisons.ts, 18, 1))
+>T : Symbol(T, Decl(deepComparisons.ts, 22, 7))
+>K : Symbol(K, Decl(deepComparisons.ts, 24, 13))
+
+        : T extends any[]
+>T : Symbol(T, Decl(deepComparisons.ts, 22, 7))
+
+            ? F<T[number]>[]
+>F : Symbol(F, Decl(deepComparisons.ts, 18, 1))
+>T : Symbol(T, Decl(deepComparisons.ts, 22, 7))
+
+            : T extends { [K: string]: any }
+>T : Symbol(T, Decl(deepComparisons.ts, 22, 7))
+>K : Symbol(K, Decl(deepComparisons.ts, 27, 27))
+
+                ? { [K in keyof T]?: F<T[K]> }
+>K : Symbol(K, Decl(deepComparisons.ts, 28, 21))
+>T : Symbol(T, Decl(deepComparisons.ts, 22, 7))
+>F : Symbol(F, Decl(deepComparisons.ts, 18, 1))
+>T : Symbol(T, Decl(deepComparisons.ts, 22, 7))
+>K : Symbol(K, Decl(deepComparisons.ts, 28, 21))
+
+                : { x: string }
+>x : Symbol(x, Decl(deepComparisons.ts, 29, 19))
+
+);
+
+declare function f<T = any>(): F<T>;
+>f : Symbol(f, Decl(deepComparisons.ts, 30, 2))
+>T : Symbol(T, Decl(deepComparisons.ts, 32, 19))
+>F : Symbol(F, Decl(deepComparisons.ts, 18, 1))
+>T : Symbol(T, Decl(deepComparisons.ts, 32, 19))
+
+function g() {
+>g : Symbol(g, Decl(deepComparisons.ts, 32, 36))
+
+    return f() as F<any>;
+>f : Symbol(f, Decl(deepComparisons.ts, 30, 2))
+>F : Symbol(F, Decl(deepComparisons.ts, 18, 1))
+}
+

tests/baselines/reference/deepComparisons.types

@@ -56,3 +56,34 @@ function f3<U>() {
 >0 as any : any
 >0 : 0
 }
+
+// Repro from #46500
+
+type F<T> = {} & (
+>F : F<T>
+
+    T extends [any, ...any[]]
+        ? { [K in keyof T]?: F<T[K]> }
+        : T extends any[]
+            ? F<T[number]>[]
+            : T extends { [K: string]: any }
+>K : string
+
+                ? { [K in keyof T]?: F<T[K]> }
+                : { x: string }
+>x : string
+
+);
+
+declare function f<T = any>(): F<T>;
+>f : <T = any>() => F<T>
+
+function g() {
+>g : () => F<any>
+
+    return f() as F<any>;
+>f() as F<any> : F<any>
+>f() : F<any>
+>f : <T = any>() => F<T>
+}
+

tests/cases/compiler/deepComparisons.ts

@@ -16,4 +16,22 @@ type Foo2<T> = { x: Foo1<T> };
 
 function f3<U>() {
     let x: Foo1<U> = 0 as any as Bar<U>;  // No error!
-}
+}
+
+// Repro from #46500
+
+type F<T> = {} & (
+    T extends [any, ...any[]]
+        ? { [K in keyof T]?: F<T[K]> }
+        : T extends any[]
+            ? F<T[number]>[]
+            : T extends { [K: string]: any }
+                ? { [K in keyof T]?: F<T[K]> }
+                : { x: string }
+);
+
+declare function f<T = any>(): F<T>;
+
+function g() {
+    return f() as F<any>;
+}