Mark the root as animating if any Portal mutates or resizes (#32772)

Portals and `<ViewTransition>` are tricky because they leave the React
tree. You might think of a Portal's container conceptually as also being
part of a React tree but that's not quite how they're modeled today.
They're more like their own roots. So instead, of trying to find a
conceptual place in the React tree we treat Portals as their own root.

We have two ways of tracking whether an update to a ViewTransition
boundary has occurred. Either a DOM mutation has happened within it, or
a resize of a child has caused it to potentially relayout its parent.
Normally that just follows the tree structure of React, but not when
it's a Portal.

When it's a Portal we don't know which DOM parent it might have
affected. For all we know it's at the root (and in fact, in most cases
that's where Portals go).

With this PR we mark the root as having been affected by a mutation or
resize. This means that the whole document will animate and we can't
optimize away from it. This ensures that a mutation to the root of a
Portal doesn't go unanimated with other things are animating such as its
parent.

You can regain this optimization by adding a `<ViewTransition>` boundary
directly inside the Portal itself so it owns its own animation. If that
DOM node is also absolutely positioned it doesn't leak.

Conversely this also means that a mutation inside a Portal doesn't
affect its React parent so it won't trigger its parent's animation if
this was the only thing animating. That could be unfortunate if this
container is actually inside the same React parent. However, because
this would have been an update we would've marked it for "maybe
animating" and updates can't only get their animations cancelled if the
root is cancelled, in practice this will actually animate anyway.

fixtures/view-transition/src/components/Page.css

@@ -21,3 +21,10 @@
   border-radius: 5px;
   padding: 10px;
 }
+
+.portal {
+  position: fixed;
+  top: 10px;
+  left: 360px;
+  border: 1px solid #ccc;
+}

fixtures/view-transition/src/components/Page.js

@@ -6,7 +6,9 @@ import React, {
   useEffect,
   useState,
   useId,
+  startTransition,
 } from 'react';
+import {createPortal} from 'react-dom';
 
 import SwipeRecognizer from './SwipeRecognizer';
 
@@ -79,6 +81,23 @@ export default function Page({url, navigate}) {
     // });
   }, [show]);
 
+  const [showModal, setShowModal] = useState(false);
+  const portal = showModal ? (
+    createPortal(
+      <div className="portal">
+        Portal: {!show ? 'A' : 'B'}
+        <ViewTransition>
+          <div>{!show ? 'A' : 'B'}</div>
+        </ViewTransition>
+      </div>,
+      document.body
+    )
+  ) : (
+    <button onClick={() => startTransition(() => setShowModal(true))}>
+      Show Modal
+    </button>
+  );
+
   const exclamation = (
     <ViewTransition name="exclamation" onShare={onTransition}>
       <span>!</span>
@@ -153,6 +172,7 @@ export default function Page({url, navigate}) {
             <p>content</p>
             <p>out</p>
             <p>of</p>
+            {portal}
             <p>the</p>
             <p>viewport</p>
             {show ? <Component /> : null}

packages/react-reconciler/src/ReactFiberCommitWork.js

@@ -283,6 +283,7 @@ export let shouldFireAfterActiveInstanceBlur: boolean = false;
 // Used during the commit phase to track whether a parent ViewTransition component
 // might have been affected by any mutations / relayouts below.
 let viewTransitionContextChanged: boolean = false;
+let rootViewTransitionAffected: boolean = false;
 
 export function commitBeforeMutationEffects(
   root: FiberRoot,
@@ -1750,6 +1751,8 @@ export function commitMutationEffects(
   inProgressLanes = committedLanes;
   inProgressRoot = root;
 
+  rootViewTransitionAffected = false;
+
   resetComponentEffectTimers();
 
   commitMutationEffectsOnFiber(finishedWork, root, committedLanes);
@@ -2068,6 +2071,7 @@ function commitMutationEffectsOnFiber(
       break;
     }
     case HostPortal: {
+      const prevMutationContext = pushMutationContext();
       if (supportsResources) {
         const previousHoistableRoot = currentHoistableRoot;
         currentHoistableRoot = getHoistableRoot(
@@ -2080,6 +2084,14 @@ function commitMutationEffectsOnFiber(
         recursivelyTraverseMutationEffects(root, finishedWork, lanes);
         commitReconciliationEffects(finishedWork, lanes);
       }
+      if (viewTransitionMutationContext) {
+        // A Portal doesn't necessarily exist within the context of this subtree.
+        // Ideally we would track which React ViewTransition component nests the container
+        // but that's costly. Instead, we treat each Portal as if it's a new React root.
+        // Therefore any leaked mutation means that the root should animate.
+        rootViewTransitionAffected = true;
+      }
+      popMutationContext(prevMutationContext);
 
       if (flags & Update) {
         if (supportsPersistence) {
@@ -2432,7 +2444,7 @@ function commitAfterMutationEffectsOnFiber(
       viewTransitionContextChanged = false;
       pushViewTransitionCancelableScope();
       recursivelyTraverseAfterMutationEffects(root, finishedWork, lanes);
-      if (!viewTransitionContextChanged) {
+      if (!viewTransitionContextChanged && !rootViewTransitionAffected) {
         // If we didn't leak any resizing out to the root, we don't have to transition
         // the root itself. This means that we can now safely cancel any cancellations
         // that bubbled all the way up.
@@ -2456,6 +2468,20 @@ function commitAfterMutationEffectsOnFiber(
       recursivelyTraverseAfterMutationEffects(root, finishedWork, lanes);
       break;
     }
+    case HostPortal: {
+      const prevContextChanged = viewTransitionContextChanged;
+      viewTransitionContextChanged = false;
+      recursivelyTraverseAfterMutationEffects(root, finishedWork, lanes);
+      if (viewTransitionContextChanged) {
+        // A Portal doesn't necessarily exist within the context of this subtree.
+        // Ideally we would track which React ViewTransition component nests the container
+        // but that's costly. Instead, we treat each Portal as if it's a new React root.
+        // Therefore any leaked resize of a child could affect the root so the root should animate.
+        rootViewTransitionAffected = true;
+      }
+      viewTransitionContextChanged = prevContextChanged;
+      break;
+    }
     case OffscreenComponent: {
       const isModernRoot =
         disableLegacyMode || (finishedWork.mode & ConcurrentMode) !== NoMode;