Retry manifest file loading only in dev mode (#73900)

In #73891 we added another manifest to be loaded in `loadComponents`
(initially unconditionally). This uncovered a flakiness in prod mode
when attempting to load an optional manifest. The non-existent manifest
is attempted to be loaded three times with 100ms delay between attempts,
before giving up. For some reason the increased loading time leads to
more test flakiness.

To mitigate this, we're limiting the retry behaviour to the dev mode,
which matches the original intention when this was introduced in #45244.

packages/next/src/build/utils.ts

@@ -1614,6 +1614,7 @@ export async function isPageStatic({
           distDir,
           page: originalAppPath || page,
           isAppPath: pageType === 'app',
+          isDev: false,
         })
       }
       const Comp = componentsResult.Component as NextComponentType | undefined
@@ -1869,6 +1870,7 @@ export async function hasCustomGetInitialProps({
     distDir,
     page: page,
     isAppPath: false,
+    isDev: false,
   })
   let mod = components.ComponentMod
 
@@ -1895,6 +1897,7 @@ export async function getDefinedNamedExports({
     distDir,
     page: page,
     isAppPath: false,
+    isDev: false,
   })
 
   return Object.keys(components.ComponentMod).filter((key) => {

packages/next/src/export/worker.ts

@@ -233,6 +233,7 @@ async function exportPageImpl(
     distDir,
     page,
     isAppPath: isAppDir,
+    isDev: false,
   })
 
   // Handle App Routes.

packages/next/src/server/dev/static-paths-worker.ts

@@ -83,6 +83,7 @@ export async function loadStaticPaths({
     // In `pages/`, the page is the same as the pathname.
     page: page || pathname,
     isAppPath,
+    isDev: true,
   })
 
   if (isAppPath) {

packages/next/src/server/load-components.ts

@@ -118,12 +118,13 @@ export async function evalManifestWithRetries<T extends object>(
 
 async function loadClientReferenceManifest(
   manifestPath: string,
-  entryName: string
+  entryName: string,
+  attempts?: number
 ) {
   try {
     const context = await evalManifestWithRetries<{
       __RSC_MANIFEST: { [key: string]: ClientReferenceManifest }
-    }>(manifestPath)
+    }>(manifestPath, attempts)
     return context.__RSC_MANIFEST[entryName]
   } catch (err) {
     return undefined
@@ -134,10 +135,12 @@ async function loadComponentsImpl<N = any>({
   distDir,
   page,
   isAppPath,
+  isDev,
 }: {
   distDir: string
   page: string
   isAppPath: boolean
+  isDev: boolean
 }): Promise<LoadComponentsReturnType<N>> {
   let DocumentMod = {}
   let AppMod = {}
@@ -151,6 +154,12 @@ async function loadComponentsImpl<N = any>({
   // Make sure to avoid loading the manifest for metadata route handlers.
   const hasClientManifest = isAppPath && !isMetadataRoute(page)
 
+  // In dev mode we retry loading a manifest file to handle a race condition
+  // that can occur while app and pages are compiling at the same time, and the
+  // build-manifest is still being written to disk while an app path is
+  // attempting to load.
+  const manifestLoadAttempts = isDev ? 3 : 1
+
   // Load the manifest files first
   const [
     buildManifest,
@@ -159,15 +168,20 @@ async function loadComponentsImpl<N = any>({
     clientReferenceManifest,
     serverActionsManifest,
   ] = await Promise.all([
-    loadManifestWithRetries<BuildManifest>(join(distDir, BUILD_MANIFEST)),
+    loadManifestWithRetries<BuildManifest>(
+      join(distDir, BUILD_MANIFEST),
+      manifestLoadAttempts
+    ),
     loadManifestWithRetries<ReactLoadableManifest>(
-      join(distDir, REACT_LOADABLE_MANIFEST)
+      join(distDir, REACT_LOADABLE_MANIFEST),
+      manifestLoadAttempts
     ),
     // This manifest will only exist in Pages dir && Production && Webpack.
     isAppPath || process.env.TURBOPACK
       ? undefined
       : loadManifestWithRetries<DynamicCssManifest>(
-          join(distDir, `${DYNAMIC_CSS_MANIFEST}.json`)
+          join(distDir, `${DYNAMIC_CSS_MANIFEST}.json`),
+          manifestLoadAttempts
         ).catch(() => undefined),
     hasClientManifest
       ? loadClientReferenceManifest(
@@ -177,12 +191,14 @@ async function loadComponentsImpl<N = any>({
             'app',
             page.replace(/%5F/g, '_') + '_' + CLIENT_REFERENCE_MANIFEST + '.js'
           ),
-          page.replace(/%5F/g, '_')
+          page.replace(/%5F/g, '_'),
+          manifestLoadAttempts
         )
       : undefined,
     isAppPath
       ? loadManifestWithRetries<ActionManifest>(
-          join(distDir, 'server', SERVER_REFERENCE_MANIFEST + '.json')
+          join(distDir, 'server', SERVER_REFERENCE_MANIFEST + '.json'),
+          manifestLoadAttempts
         ).catch(() => null)
       : null,
   ])

packages/next/src/server/next-server.ts

@@ -177,11 +177,14 @@ export default class NextNodeServer extends BaseServer<
 
   protected cleanupListeners = new AsyncCallbackSet()
   protected internalWaitUntil: WaitUntil | undefined
+  private isDev: boolean
 
   constructor(options: Options) {
     // Initialize super class
     super(options)
 
+    this.isDev = options.dev ?? false
+
     /**
      * This sets environment variable to be used at the time of SSR by head.tsx.
      * Using this from process.env allows targeting SSR by calling
@@ -213,11 +216,13 @@ export default class NextNodeServer extends BaseServer<
         distDir: this.distDir,
         page: '/_document',
         isAppPath: false,
+        isDev: this.isDev,
       }).catch(() => {})
       loadComponents({
         distDir: this.distDir,
         page: '/_app',
         isAppPath: false,
+        isDev: this.isDev,
       }).catch(() => {})
     }
 
@@ -278,11 +283,17 @@ export default class NextNodeServer extends BaseServer<
         distDir: this.distDir,
         page,
         isAppPath: false,
+        isDev: this.isDev,
       }).catch(() => {})
     }
 
     for (const page of Object.keys(appPathsManifest || {})) {
-      await loadComponents({ distDir: this.distDir, page, isAppPath: true })
+      await loadComponents({
+        distDir: this.distDir,
+        page,
+        isAppPath: true,
+        isDev: this.isDev,
+      })
         .then(async ({ ComponentMod }) => {
           // we need to ensure fetch is patched before we require the page,
           // otherwise if the fetch is patched by user code, we will be patching it
@@ -789,6 +800,7 @@ export default class NextNodeServer extends BaseServer<
           distDir: this.distDir,
           page: pagePath,
           isAppPath,
+          isDev: this.isDev,
         })
 
         if (