src: add an option to make compile cache portable

Adds an option (NODE_COMPILE_CACHE_PORTABLE) for
the built-in compile cache to encode the hashes with
relative file paths. On enabling the option,
the source directory along with cache directory can be
bundled and moved, and the cache continues to work.

When enabled, paths encoded in hash are relative to
compile cache directory.

PR-URL: https://github.com/nodejs/node/pull/58797
Fixes: https://github.com/nodejs/node/issues/58755
Refs: https://github.com/nodejs/node/issues/52696
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>

doc/api/cli.md

@@ -3341,6 +3341,11 @@ added: v22.1.0
 Enable the [module compile cache][] for the Node.js instance. See the documentation of
 [module compile cache][] for details.
 
+### `NODE_COMPILE_CACHE_PORTABLE=1`
+
+When set to 1, the [module compile cache][]  can be reused across different directory
+locations as long as the module layout relative to the cache directory remains the same.
+
 ### `NODE_DEBUG=module[,…]`
 
 <!-- YAML

doc/api/module.md

@@ -399,6 +399,28 @@ the [`NODE_COMPILE_CACHE=dir`][] environment variable if it's set, or defaults
 to `path.join(os.tmpdir(), 'node-compile-cache')` otherwise. To locate the compile cache
 directory used by a running Node.js instance, use [`module.getCompileCacheDir()`][].
 
+By default, caches are invalidated when the absolute paths of the modules being
+cached are changed. To keep the cache working after moving the
+project directory, enable portable compile cache. This allows previously compiled
+modules to be reused across different directory locations as long as the layout relative
+to the cache directory remains the same. This would be done on a best-effort basis. If
+Node.js cannot compute the location of a module relative to the cache directory, the module
+will not be cached.
+
+There are two ways to enable the portable mode:
+
+1. Using the portable option in module.enableCompileCache():
+
+   ```js
+   // Non-portable cache (default): cache breaks if project is moved
+   module.enableCompileCache({ path: '/path/to/cache/storage/dir' });
+
+   // Portable cache: cache works after the project is moved
+   module.enableCompileCache({ path: '/path/to/cache/storage/dir', portable: true });
+   ```
+
+2. Setting the environment variable: [`NODE_COMPILE_CACHE_PORTABLE=1`][]
+
 Currently when using the compile cache with [V8 JavaScript code coverage][], the
 coverage being collected by V8 may be less precise in functions that are
 deserialized from the code cache. It's recommended to turn this off when
@@ -1789,6 +1811,7 @@ returned object contains the following keys:
 [`--import`]: cli.md#--importmodule
 [`--require`]: cli.md#-r---require-module
 [`NODE_COMPILE_CACHE=dir`]: cli.md#node_compile_cachedir
+[`NODE_COMPILE_CACHE_PORTABLE=1`]: cli.md#node_compile_cache_portable1
 [`NODE_DISABLE_COMPILE_CACHE=1`]: cli.md#node_disable_compile_cache1
 [`NODE_V8_COVERAGE=dir`]: cli.md#node_v8_coveragedir
 [`SourceMap`]: #class-modulesourcemap

doc/node.1

@@ -725,6 +725,13 @@ Enable the
 .Sy module compile cache
 for the Node.js instance.
 .
+.It Ev NODE_COMPILE_CACHE_PORTABLE
+When set to '1' or 'true', the
+.Sy module compile cache
+will be hit as long as the location of the modules relative to the cache directory remain
+consistent. This can be used in conjunction with .Ev NODE_COMPILE_CACHE
+to enable portable on-disk caching.
+.
 .It Ev NODE_DEBUG Ar modules...
 Comma-separated list of core modules that should print debug information.
 .

lib/internal/modules/helpers.js

@@ -402,18 +402,31 @@ function stringify(body) {
 }
 
 /**
- * Enable on-disk compiled cache for all user modules being complied in the current Node.js instance
+ * Enable on-disk compiled cache for all user modules being compiled in the current Node.js instance
  * after this method is called.
- * If cacheDir is undefined, defaults to the NODE_MODULE_CACHE environment variable.
- * If NODE_MODULE_CACHE isn't set, default to path.join(os.tmpdir(), 'node-compile-cache').
- * @param {string|undefined} cacheDir
+ * This method accepts either:
+ * - A string `cacheDir`: the path to the cache directory.
+ * - An options object `{path?: string, portable?: boolean}`:
+ *   - `path`: A string path to the cache directory.
+ *   - `portable`: If `portable` is true, the cache directory will be considered relative. Defaults to false.
+ * If cache path is undefined, it defaults to the NODE_MODULE_CACHE environment variable.
+ * If `NODE_MODULE_CACHE` isn't set, it defaults to `path.join(os.tmpdir(), 'node-compile-cache')`.
+ * @param {string | { path?: string, portable?: boolean } | undefined} options
  * @returns {{status: number, message?: string, directory?: string}}
  */
-function enableCompileCache(cacheDir) {
+function enableCompileCache(options) {
+  let cacheDir;
+  let portable = false;
+
+  if (typeof options === 'object' && options !== null) {
+    ({ path: cacheDir, portable = false } = options);
+  } else {
+    cacheDir = options;
+  }
   if (cacheDir === undefined) {
     cacheDir = join(lazyTmpdir(), 'node-compile-cache');
   }
-  const nativeResult = _enableCompileCache(cacheDir);
+  const nativeResult = _enableCompileCache(cacheDir, portable);
   const result = { status: nativeResult[0] };
   if (nativeResult[1]) {
     result.message = nativeResult[1];

src/compile_cache.cc

@@ -13,6 +13,9 @@
 #include <unistd.h>  // getuid
 #endif
 
+#ifdef _WIN32
+#include <windows.h>
+#endif
 namespace node {
 
 using v8::Function;
@@ -223,13 +226,52 @@ void CompileCacheHandler::ReadCacheFile(CompileCacheEntry* entry) {
   Debug(" success, size=%d\n", total_read);
 }
 
+static std::string GetRelativePath(std::string_view path,
+                                   std::string_view base) {
+// On Windows, the native encoding is UTF-16, so we need to convert
+// the paths to wide strings before using std::filesystem::path.
+// On other platforms, std::filesystem::path can handle UTF-8 directly.
+#ifdef _WIN32
+  std::filesystem::path module_path(
+      ConvertToWideString(std::string(path), CP_UTF8));
+  std::filesystem::path base_path(
+      ConvertToWideString(std::string(base), CP_UTF8));
+#else
+  std::filesystem::path module_path(path);
+  std::filesystem::path base_path(base);
+#endif
+  std::filesystem::path relative = module_path.lexically_relative(base_path);
+  auto u8str = relative.u8string();
+  return std::string(u8str.begin(), u8str.end());
+}
+
 CompileCacheEntry* CompileCacheHandler::GetOrInsert(Local<String> code,
                                                     Local<String> filename,
                                                     CachedCodeType type) {
   DCHECK(!compile_cache_dir_.empty());
 
+  Environment* env = Environment::GetCurrent(isolate_->GetCurrentContext());
   Utf8Value filename_utf8(isolate_, filename);
-  uint32_t key = GetCacheKey(filename_utf8.ToStringView(), type);
+  std::string file_path = filename_utf8.ToString();
+  // If the portable cache is enabled and it seems possible to compute the
+  // relative position from an absolute path, we use the relative position
+  // in the cache key.
+  if (portable_ == EnableOption::PORTABLE && IsAbsoluteFilePath(file_path)) {
+    // Normalize the path to ensure it is consistent.
+    std::string normalized_file_path = NormalizeFileURLOrPath(env, file_path);
+    if (normalized_file_path.empty()) {
+      return nullptr;
+    }
+    std::string relative_path =
+        GetRelativePath(normalized_file_path, normalized_compile_cache_dir_);
+    if (!relative_path.empty()) {
+      file_path = relative_path;
+      Debug("[compile cache] using relative path %s from %s\n",
+            file_path.c_str(),
+            compile_cache_dir_.c_str());
+    }
+  }
+  uint32_t key = GetCacheKey(file_path, type);
 
   // TODO(joyeecheung): don't encode this again into UTF8. If we read the
   // UTF8 content on disk as raw buffer (from the JS layer, while watching out
@@ -500,7 +542,8 @@ CompileCacheHandler::CompileCacheHandler(Environment* env)
 //   - $NODE_VERSION-$ARCH-$CACHE_DATA_VERSION_TAG-$UID
 //     - $FILENAME_AND_MODULE_TYPE_HASH.cache: a hash of filename + module type
 CompileCacheEnableResult CompileCacheHandler::Enable(Environment* env,
-                                                     const std::string& dir) {
+                                                     const std::string& dir,
+                                                     EnableOption option) {
   std::string cache_tag = GetCacheVersionTag();
   std::string absolute_cache_dir_base = PathResolve(env, {dir});
   std::string cache_dir_with_tag =
@@ -548,6 +591,11 @@ CompileCacheEnableResult CompileCacheHandler::Enable(Environment* env,
 
   result.cache_directory = absolute_cache_dir_base;
   compile_cache_dir_ = cache_dir_with_tag;
+  portable_ = option;
+  if (option == EnableOption::PORTABLE) {
+    normalized_compile_cache_dir_ =
+        NormalizeFileURLOrPath(env, compile_cache_dir_);
+  }
   result.status = CompileCacheEnableStatus::ENABLED;
   return result;
 }

src/compile_cache.h

@@ -62,10 +62,14 @@ struct CompileCacheEnableResult {
   std::string message;  // Set in case of failure.
 };
 
+enum class EnableOption : uint8_t { DEFAULT, PORTABLE };
+
 class CompileCacheHandler {
  public:
   explicit CompileCacheHandler(Environment* env);
-  CompileCacheEnableResult Enable(Environment* env, const std::string& dir);
+  CompileCacheEnableResult Enable(Environment* env,
+                                  const std::string& dir,
+                                  EnableOption option = EnableOption::DEFAULT);
 
   void Persist();
 
@@ -103,6 +107,8 @@ class CompileCacheHandler {
   bool is_debug_ = false;
 
   std::string compile_cache_dir_;
+  std::string normalized_compile_cache_dir_;
+  EnableOption portable_ = EnableOption::DEFAULT;
   std::unordered_map<uint32_t, std::unique_ptr<CompileCacheEntry>>
       compiler_cache_store_;
 };

src/env.cc

@@ -1128,11 +1128,21 @@ void Environment::InitializeCompileCache() {
       dir_from_env.empty()) {
     return;
   }
-  EnableCompileCache(dir_from_env);
+  std::string portable_env;
+  bool portable = credentials::SafeGetenv(
+                      "NODE_COMPILE_CACHE_PORTABLE", &portable_env, this) &&
+                  !portable_env.empty() && portable_env == "1";
+  if (portable) {
+    Debug(this,
+          DebugCategory::COMPILE_CACHE,
+          "[compile cache] using relative path\n");
+  }
+  EnableCompileCache(dir_from_env,
+                     portable ? EnableOption::PORTABLE : EnableOption::DEFAULT);
 }
 
 CompileCacheEnableResult Environment::EnableCompileCache(
-    const std::string& cache_dir) {
+    const std::string& cache_dir, EnableOption option) {
   CompileCacheEnableResult result;
   std::string disable_env;
   if (credentials::SafeGetenv(
@@ -1149,7 +1159,7 @@ CompileCacheEnableResult Environment::EnableCompileCache(
   if (!compile_cache_handler_) {
     std::unique_ptr<CompileCacheHandler> handler =
         std::make_unique<CompileCacheHandler>(this);
-    result = handler->Enable(this, cache_dir);
+    result = handler->Enable(this, cache_dir, option);
     if (result.status == CompileCacheEnableStatus::ENABLED) {
       compile_cache_handler_ = std::move(handler);
       AtExit(

src/env.h

@@ -1020,7 +1020,8 @@ class Environment final : public MemoryRetainer {
   void InitializeCompileCache();
   // Enable built-in compile cache if it has not yet been enabled.
   // The cache will be persisted to disk on exit.
-  CompileCacheEnableResult EnableCompileCache(const std::string& cache_dir);
+  CompileCacheEnableResult EnableCompileCache(const std::string& cache_dir,
+                                              EnableOption option);
   void FlushCompileCache();
 
   void RunAndClearNativeImmediates(bool only_refed = false);

src/node_modules.cc

@@ -501,8 +501,14 @@ void EnableCompileCache(const FunctionCallbackInfo<Value>& args) {
     THROW_ERR_INVALID_ARG_TYPE(env, "cacheDir should be a string");
     return;
   }
+
+  EnableOption option = EnableOption::DEFAULT;
+  if (args.Length() > 1 && args[1]->IsTrue()) {
+    option = EnableOption::PORTABLE;
+  }
+
   Utf8Value value(isolate, args[0]);
-  CompileCacheEnableResult result = env->EnableCompileCache(*value);
+  CompileCacheEnableResult result = env->EnableCompileCache(*value, option);
   Local<Value> values[3];
   values[0] = v8::Integer::New(isolate, static_cast<uint8_t>(result.status));
   if (ToV8Value(context, result.message).ToLocal(&values[1]) &&

src/path.cc

@@ -1,8 +1,10 @@
 #include "path.h"
 #include <string>
 #include <vector>
+#include "ada.h"
 #include "env-inl.h"
 #include "node_internals.h"
+#include "node_url.h"
 
 namespace node {
 
@@ -88,6 +90,10 @@ std::string NormalizeString(const std::string_view path,
 }
 
 #ifdef _WIN32
+constexpr bool IsWindowsDriveLetter(const std::string_view path) noexcept {
+  return path.size() > 2 && IsWindowsDeviceRoot(path[0]) &&
+         (path[1] == ':' && (path[2] == '/' || path[2] == '\\'));
+}
 constexpr bool IsWindowsDeviceRoot(const char c) noexcept {
   return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
 }
@@ -333,4 +339,44 @@ void FromNamespacedPath(std::string* path) {
 #endif
 }
 
+// Check if a path looks like an absolute path or file URL.
+bool IsAbsoluteFilePath(std::string_view path) {
+  if (path.rfind("file://", 0) == 0) {
+    return true;
+  }
+#ifdef _WIN32
+  if (path.size() > 0 && path[0] == '\\') return true;
+  if (IsWindowsDriveLetter(path)) return true;
+#endif
+  if (path.size() > 0 && path[0] == '/') return true;
+  return false;
+}
+
+// Normalizes paths by resolving file URLs and converting to a consistent
+// format with forward slashes.
+std::string NormalizeFileURLOrPath(Environment* env, std::string_view path) {
+  std::string normalized_string(path);
+  constexpr std::string_view file_scheme = "file://";
+  if (normalized_string.rfind(file_scheme, 0) == 0) {
+    auto out = ada::parse<ada::url_aggregator>(normalized_string);
+    auto file_path = url::FileURLToPath(env, *out);
+    if (!file_path.has_value()) {
+      return std::string();
+    }
+    normalized_string = file_path.value();
+  }
+  normalized_string = NormalizeString(normalized_string, false, "/");
+#ifdef _WIN32
+  if (IsWindowsDriveLetter(normalized_string)) {
+    normalized_string[0] = ToLower(normalized_string[0]);
+  }
+  for (char& c : normalized_string) {
+    if (c == '\\') {
+      c = '/';
+    }
+  }
+#endif
+  return normalized_string;
+}
+
 }  // namespace node

src/path.h

@@ -18,9 +18,12 @@ std::string NormalizeString(const std::string_view path,
 
 std::string PathResolve(Environment* env,
                         const std::vector<std::string_view>& paths);
+std::string NormalizeFileURLOrPath(Environment* env, std::string_view path);
+bool IsAbsoluteFilePath(std::string_view path);
 
 #ifdef _WIN32
 constexpr bool IsWindowsDeviceRoot(const char c) noexcept;
+constexpr bool IsWindowsDriveLetter(const std::string_view path) noexcept;
 #endif  // _WIN32
 
 void ToNamespacedPath(Environment* env, BufferValue* path);

test/parallel/test-compile-cache-api-error.js

@@ -6,6 +6,6 @@ require('../common');
 const { enableCompileCache } = require('module');
 const assert = require('assert');
 
-for (const invalid of [0, null, false, () => {}, {}, []]) {
+for (const invalid of [0, null, false, 1, NaN, true, Symbol(0)]) {
   assert.throws(() => enableCompileCache(invalid), { code: 'ERR_INVALID_ARG_TYPE' });
 }

test/parallel/test-compile-cache-api-portable.js

@@ -0,0 +1,106 @@
+'use strict';
+
+// This tests module.enableCompileCache({ path, portable: true }) works
+// and supports portable paths across directory relocations.
+
+require('../common');
+const { spawnSyncAndAssert } = require('../common/child_process');
+const assert = require('assert');
+const fs = require('fs');
+const tmpdir = require('../common/tmpdir');
+const path = require('path');
+
+tmpdir.refresh();
+const workDir = path.join(tmpdir.path, 'work');
+const cacheRel = '.compile_cache_dir';
+fs.mkdirSync(workDir, { recursive: true });
+
+const wrapper = path.join(workDir, 'wrapper.js');
+const target = path.join(workDir, 'target.js');
+
+fs.writeFileSync(
+  wrapper,
+  `
+  const { enableCompileCache, getCompileCacheDir } = require('module');
+  console.log('dir before enableCompileCache:', getCompileCacheDir());
+  enableCompileCache({ path: '${cacheRel}', portable: true });
+  console.log('dir after enableCompileCache:', getCompileCacheDir());
+`
+);
+
+fs.writeFileSync(target, '');
+
+// First run
+{
+  spawnSyncAndAssert(
+    process.execPath,
+    ['-r', wrapper, target],
+    {
+      env: {
+        ...process.env,
+        NODE_DEBUG_NATIVE: 'COMPILE_CACHE',
+      },
+      cwd: workDir,
+    },
+    {
+      stdout(output) {
+        console.log(output);
+        assert.match(output, /dir before enableCompileCache: undefined/);
+        assert.match(
+          output,
+          /dir after enableCompileCache: .+\.compile_cache_dir/
+        );
+        return true;
+      },
+      stderr(output) {
+        assert.match(
+          output,
+          /target\.js was not initialized, initializing the in-memory entry/
+        );
+        assert.match(output, /writing cache for .*target\.js.*success/);
+        return true;
+      },
+    }
+  );
+}
+
+// Second run — moved directory, but same relative cache path
+{
+  const movedWorkDir = `${workDir}_moved`;
+  fs.renameSync(workDir, movedWorkDir);
+
+  spawnSyncAndAssert(
+    process.execPath,
+    [
+      '-r',
+      path.join(movedWorkDir, 'wrapper.js'),
+      path.join(movedWorkDir, 'target.js'),
+    ],
+    {
+      env: {
+        ...process.env,
+        NODE_DEBUG_NATIVE: 'COMPILE_CACHE',
+      },
+      cwd: movedWorkDir,
+    },
+    {
+      stdout(output) {
+        console.log(output);
+        assert.match(output, /dir before enableCompileCache: undefined/);
+        assert.match(
+          output,
+          /dir after enableCompileCache: .+\.compile_cache_dir/
+        );
+        return true;
+      },
+      stderr(output) {
+        assert.match(
+          output,
+          /cache for .*target\.js was accepted, keeping the in-memory entry/
+        );
+        assert.match(output, /.*skip .*target\.js because cache was the same/);
+        return true;
+      },
+    }
+  );
+}

test/parallel/test-compile-cache-portable-esm.js

@@ -0,0 +1,84 @@
+'use strict';
+
+// This tests NODE_COMPILE_CACHE works after moving directory and unusual characters in path are handled correctly.
+
+require('../common');
+const { spawnSyncAndAssert } = require('../common/child_process');
+const assert = require('assert');
+const tmpdir = require('../common/tmpdir');
+const fs = require('fs');
+const path = require('path');
+
+tmpdir.refresh();
+
+const workDir = path.join(tmpdir.path, 'work');
+const cacheRel = '.compile_cache_dir';
+fs.mkdirSync(workDir, { recursive: true });
+
+const script = path.join(workDir, 'message.mjs');
+fs.writeFileSync(
+  script,
+  `
+  export const message = 'A message';
+  `
+);
+
+{
+  spawnSyncAndAssert(
+    process.execPath,
+    [script],
+    {
+      env: {
+        ...process.env,
+        NODE_DEBUG_NATIVE: 'COMPILE_CACHE',
+        NODE_COMPILE_CACHE: cacheRel,
+        NODE_COMPILE_CACHE_PORTABLE: '1',
+      },
+      cwd: workDir,
+    },
+    {
+      stderr(output) {
+        console.log(output);
+        assert.match(
+          output,
+          /message\.mjs was not initialized, initializing the in-memory entry/
+        );
+        assert.match(output, /writing cache for .*message\.mjs.*success/);
+        return true;
+      },
+    }
+  );
+
+  // Move the working directory and run again
+  const movedWorkDir = `${workDir}_moved`;
+  fs.renameSync(workDir, movedWorkDir);
+
+
+  spawnSyncAndAssert(
+    process.execPath,
+    [[path.join(movedWorkDir, 'message.mjs')]],
+    {
+      env: {
+        ...process.env,
+        NODE_DEBUG_NATIVE: 'COMPILE_CACHE',
+        NODE_COMPILE_CACHE: cacheRel,
+        NODE_COMPILE_CACHE_PORTABLE: '1',
+      },
+      cwd: movedWorkDir,
+    },
+    {
+      stderr(output) {
+        console.log(output);
+        assert.match(
+          output,
+          /cache for .*message\.mjs was accepted, keeping the in-memory entry/
+        );
+        assert.match(
+          output,
+          /.*skip .*message\.mjs because cache was the same/
+        );
+        return true;
+      },
+    }
+  );
+}

test/parallel/test-compile-cache-portable.js

@@ -0,0 +1,75 @@
+'use strict';
+
+// This tests NODE_COMPILE_CACHE works with the NODE_COMPILE_CACHE_PORTABLE
+// environment variable.
+
+require('../common');
+const { spawnSyncAndAssert } = require('../common/child_process');
+const assert = require('assert');
+const fs = require('fs');
+const tmpdir = require('../common/tmpdir');
+const path = require('path');
+
+tmpdir.refresh();
+const workDir = path.join(tmpdir.path, 'work');
+const cacheRel = '.compile_cache_dir';
+fs.mkdirSync(workDir, { recursive: true });
+
+const script = path.join(workDir, 'script.js');
+fs.writeFileSync(script, '');
+
+// First run
+{
+  spawnSyncAndAssert(
+    process.execPath,
+    [script],
+    {
+      env: {
+        ...process.env,
+        NODE_DEBUG_NATIVE: 'COMPILE_CACHE',
+        NODE_COMPILE_CACHE: cacheRel,
+        NODE_COMPILE_CACHE_PORTABLE: '1',
+      },
+      cwd: workDir,
+    },
+    {
+      stderr(output) {
+        console.log(output);
+        assert.match(
+          output,
+          /script\.js was not initialized, initializing the in-memory entry/
+        );
+        assert.match(output, /writing cache for .*script\.js.*success/);
+        return true;
+      },
+    }
+  );
+
+  // Move the working directory and run again
+  const movedWorkDir = `${workDir}_moved`;
+  fs.renameSync(workDir, movedWorkDir);
+  spawnSyncAndAssert(
+    process.execPath,
+    [[path.join(movedWorkDir, 'script.js')]],
+    {
+      env: {
+        ...process.env,
+        NODE_DEBUG_NATIVE: 'COMPILE_CACHE',
+        NODE_COMPILE_CACHE: cacheRel,
+        NODE_COMPILE_CACHE_PORTABLE: '1',
+      },
+      cwd: movedWorkDir,
+    },
+    {
+      stderr(output) {
+        console.log(output);
+        assert.match(
+          output,
+          /cache for .*script\.js was accepted, keeping the in-memory entry/
+        );
+        assert.match(output, /.*skip .*script\.js because cache was the same/);
+        return true;
+      },
+    }
+  );
+}