tools: run CI with shared libs on GHA

PR-URL: https://github.com/nodejs/node/pull/60121
Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com>
Reviewed-By: Chengzhong Wu <legendecas@gmail.com>

.github/workflows/linters.yml

@@ -140,6 +140,22 @@ jobs:
           NODE=$(command -v node) make lint-md
         env:
           NODE_RELEASED_VERSIONS: ${{ steps.get-released-versions.outputs.NODE_RELEASED_VERSIONS }}
+  lint-nix:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8  # v5.0.0
+        with:
+          persist-credentials: false
+          sparse-checkout: '*.nix'
+          sparse-checkout-cone-mode: false
+      - uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487  # v31.6.1
+      - name: Lint Nix files
+        run: nix-shell -I nixpkgs=./tools/nix/pkgs.nix -p 'nixfmt-tree' --run 'treefmt --quiet --fail-on-change'
+      - if: ${{ failure() }}
+        name: Show diff
+        run: git --no-pager diff
+
   lint-py:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest

.github/workflows/test-linux.yml

@@ -5,6 +5,7 @@ on:
     paths-ignore:
       - .mailmap
       - README.md
+      - '*.nix'
       - .github/**
       - '!.github/workflows/test-linux.yml'
     types: [opened, synchronize, reopened, ready_for_review]
@@ -17,6 +18,7 @@ on:
     paths-ignore:
       - .mailmap
       - README.md
+      - '*.nix'
       - .github/**
       - '!.github/workflows/test-linux.yml'
 

.github/workflows/test-macos.yml

@@ -6,6 +6,7 @@ on:
     paths-ignore:
       - .mailmap
       - '**.md'
+      - '*.nix'
       - AUTHORS
       - doc/**
       - .github/**
@@ -19,6 +20,7 @@ on:
     paths-ignore:
       - .mailmap
       - '**.md'
+      - '*.nix'
       - AUTHORS
       - doc/**
       - .github/**

.github/workflows/test-shared.yml

@@ -0,0 +1,83 @@
+name: Test Shared librairies
+
+on:
+  pull_request:
+    paths-ignore:
+      - .mailmap
+      - '**.md'
+      - AUTHORS
+      - doc/**
+      - .github/**
+      - '!.github/workflows/test-shared.yml'
+    types: [opened, synchronize, reopened, ready_for_review]
+  push:
+    branches:
+      - main
+      - canary
+      - v[0-9]+.x-staging
+      - v[0-9]+.x
+    paths-ignore:
+      - .mailmap
+      - '**.md'
+      - AUTHORS
+      - doc/**
+      - .github/**
+      - '!.github/workflows/test-shared.yml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  FLAKY_TESTS: keep_retrying
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            system: x86_64-linux
+          - runner: ubuntu-24.04-arm
+            system: aarch64-linux
+          - runner: macos-13
+            system: x86_64-darwin
+          - runner: macos-latest
+            system: aarch64-darwin
+    name: '${{ matrix.system }}: with shared libraries'
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        with:
+          persist-credentials: false
+
+      - uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487  # v31.6.1
+        with:
+          extra_nix_config: sandbox = true
+
+      - name: Configure sccache
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1
+        with:
+          script: |
+            core.exportVariable('SCCACHE_GHA_VERSION', 'on');
+            core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on');
+            core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
+      - name: Build Node.js and run tests
+        run: |
+          nix-shell \
+            -I nixpkgs=./tools/nix/pkgs.nix \
+            --pure --keep FLAKY_TESTS \
+            --keep SCCACHE_GHA_VERSION --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \
+            --arg loadJSBuiltinsDynamically false \
+            --arg ccache '(import <nixpkgs> {}).sccache' \
+            --arg devTools '[]' \
+            --arg benchmarkTools '[]' \
+            --run '
+                make run-ci -j4 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9 --skip-tests=$CI_SKIP_TESTS"
+            '

.github/workflows/tools.yml

@@ -30,6 +30,7 @@ on:
           - llhttp
           - minimatch
           - nbytes
+          - nixpkgs-unstable
           - nghttp2
           - nghttp3
           - ngtcp2
@@ -190,6 +191,14 @@ jobs:
               cat temp-output
               tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true
               rm temp-output
+          - id: nixpkgs-unstable
+            subsystem: tools
+            label: tools
+            run: |
+              ./tools/dep_updaters/update-nixpkgs-pin.sh > temp-output
+              cat temp-output
+              tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true
+              rm temp-output
           - id: nghttp2
             subsystem: deps
             label: dependencies
@@ -292,6 +301,9 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           allow-prereleases: true
+      - name: Set up Nix
+        if: matrix.id == 'nixpkgs-unstable' && (github.event_name == 'schedule' || inputs.id == 'all' || inputs.id == matrix.id)
+        uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487  # v31.6.1
       - run: ${{ matrix.run }}
         if: github.event_name == 'schedule' || inputs.id == 'all' || inputs.id == matrix.id
         env:

BUILDING.md

@@ -239,6 +239,7 @@ Consult previous versions of this document for older versions of Node.js:
 
 Installation via Linux package manager can be achieved with:
 
+* Nix, NixOS: `nix-shell`
 * Ubuntu, Debian: `sudo apt-get install python3 g++-12 gcc-12 make python3-pip`
 * Fedora: `sudo dnf install python3 gcc-c++ make python3-pip`
 * CentOS and RHEL: `sudo yum install python3 gcc-c++ make python3-pip`
@@ -259,6 +260,75 @@ installed, you can find them under the menu `Xcode -> Open Developer Tool ->
 More Developer Tools...`. This step will install `clang`, `clang++`, and
 `make`.
 
+#### Nix integration
+
+If you are using Nix and direnv, you can use the following to get started:
+
+```bash
+echo 'use_nix --arg sharedLibDeps {} --argstr icu small' > .envrc
+direnv allow .
+make build-ci -j12
+```
+
+The use of `make build-ci` is to ensure you are using the `CONFIG_FLAGS`
+environment variable. You can also specify it manually:
+
+```bash
+./configure $CONFIG_FLAGS
+make -j12
+```
+
+Passing the `--arg sharedLibDeps {}` instructs direnv and Nix to generate an
+environment that uses the vendored-in native dependencies. Using the vendored-in
+dependencies result in a result closer to the official binaries, the tradeoff
+being the build will take longer to complete as you'd have to build those
+dependencies instead of using the cached ones from the Nix cache. You can omit
+that flag to use all the shared dependencies, or specify only some dependencies:
+
+```bash
+cat -> .envrc <<'EOF'
+use nix --arg sharedLibDeps '{
+  inherit (import <nixpkgs> {})
+    openssl
+    zlib
+  ;
+}'
+EOF
+```
+
+Passing the `--argstr icu small` instructs direnv and Nix to pass `--with-intl=small` in
+the `CONFIG_FLAGS` environment variable. If you omit this, the prebuilt ICU from Nix cache
+will be used, which should speed up greatly compilation time.
+
+The use of `direnv` is completely optional, you can also use `nix-shell` directly,
+e.g. here's a command you can use to build a binary for benchmarking purposes:
+
+```bash
+# Passing `--arg loadJSBuiltinsDynamically false` to instruct the compiler to
+# embed the JS core files so it is no longer affected by local changes
+# (necessary for getting useful benchmark results).
+# Passing `--arg devTools '[]' --arg benchmarkTools '[]'` since we don't need
+# those to build node.
+nix-shell \
+  --arg loadJSBuiltinsDynamically false \
+  --arg devTools '[]' --arg benchmarkTools '[]' \
+  --run 'make build-ci -j12'
+
+mv out/Release/node ./node_old
+
+# ...
+# Make your local changes, and re-build node
+
+nix-shell \
+  --arg loadJSBuiltinsDynamically false \
+  --arg devTools '[]' --arg benchmarkTools '[]' \
+  --run 'make build-ci -j12'
+
+nix-shell --pure --run './node benchmark/compare.js --old ./node_old  --new ./node http | Rscript benchmark/compare.R'
+```
+
+There are additional attributes you can pass, see `shell.nix` file for more details.
+
 #### Building Node.js
 
 If the path to your build directory contains a space, the build will likely
@@ -267,7 +337,6 @@ fail.
 To build Node.js:
 
 ```bash
-export CXX=g++-12
 ./configure
 make -j4
 ```

doc/contributing/writing-and-running-benchmarks.md

@@ -26,6 +26,10 @@ Basic Unix tools are required for some benchmarks.
 [Git for Windows][git-for-windows] includes Git Bash and the necessary tools,
 which need to be included in the global Windows `PATH`.
 
+If you are using Nix, all the required tools are already listed in the
+`benchmarkTools` argument of the `shell.nix` file, so you can skip those
+prerequesites.
+
 ### HTTP benchmark requirements
 
 Most of the HTTP benchmarks require a benchmarker to be installed. This can be

shell.nix

@@ -0,0 +1,120 @@
+{
+  pkgs ? import "${./tools/nix/pkgs.nix}" { },
+  loadJSBuiltinsDynamically ? true, # Load `lib/**.js` from disk instead of embedding
+  ncu-path ? null, # Provide this if you want to use a local version of NCU
+  icu ? pkgs.icu,
+  sharedLibDeps ? {
+    inherit (pkgs)
+      ada
+      brotli
+      c-ares
+      libuv
+      nghttp2
+      nghttp3
+      ngtcp2
+      openssl
+      simdjson
+      simdutf
+      sqlite
+      uvwasi
+      zlib
+      zstd
+      ;
+    http-parser = pkgs.llhttp;
+  },
+  ccache ? pkgs.ccache,
+  ninja ? pkgs.ninja,
+  devTools ? [
+    pkgs.curl
+    pkgs.gh
+    pkgs.git
+    pkgs.jq
+    pkgs.shellcheck
+  ]
+  ++ (
+    if (ncu-path == null) then
+      [ pkgs.node-core-utils ]
+    else
+      [
+        (pkgs.writeShellScriptBin "git-node" "exec \"${ncu-path}/bin/git-node.js\" \"$@\"")
+        (pkgs.writeShellScriptBin "ncu-ci" "exec \"${ncu-path}/bin/ncu-ci.js\" \"$@\"")
+        (pkgs.writeShellScriptBin "ncu-config" "exec \"${ncu-path}/bin/ncu-config.js\" \"$@\"")
+      ]
+  ),
+  benchmarkTools ? [
+    pkgs.R
+    pkgs.rPackages.ggplot2
+    pkgs.rPackages.plyr
+    pkgs.wrk
+  ],
+  extraConfigFlags ? [
+    "--without-npm"
+    "--debug-node"
+  ],
+}:
+
+let
+  useSharedICU = if builtins.isString icu then icu == "system" else icu != null;
+  useSharedAda = builtins.hasAttr "ada" sharedLibDeps;
+  useSharedOpenSSL = builtins.hasAttr "openssl" sharedLibDeps;
+in
+pkgs.mkShell {
+  inherit (pkgs.nodejs_latest) nativeBuildInputs;
+
+  buildInputs = builtins.attrValues sharedLibDeps ++ pkgs.lib.optionals useSharedICU [ icu ];
+
+  packages = [
+    ccache
+  ]
+  ++ devTools
+  ++ benchmarkTools;
+
+  shellHook =
+    if (ccache != null) then
+      ''
+        export CC="${pkgs.lib.getExe ccache} $CC"
+        export CXX="${pkgs.lib.getExe ccache} $CXX"
+      ''
+    else
+      "";
+
+  BUILD_WITH = if (ninja != null) then "ninja" else "make";
+  NINJA = if (ninja != null) then "${pkgs.lib.getExe ninja}" else "";
+  CI_SKIP_TESTS = pkgs.lib.concatStringsSep "," (
+    [ ]
+    ++ pkgs.lib.optionals useSharedAda [
+      # Different versions of Ada affect the WPT tests
+      "test-url"
+    ]
+    ++ pkgs.lib.optionals useSharedOpenSSL [
+      # Path to the openssl.cnf is different from the expected one
+      "test-strace-openat-openssl"
+    ]
+  );
+  CONFIG_FLAGS = builtins.toString (
+    [
+      (
+        if icu == null then
+          "--without-intl"
+        else
+          "--with-intl=${if useSharedICU then "system" else icu}-icu"
+      )
+    ]
+    ++ extraConfigFlags
+    ++ pkgs.lib.optionals (ninja != null) [
+      "--ninja"
+    ]
+    ++ pkgs.lib.optionals loadJSBuiltinsDynamically [
+      "--node-builtin-modules-path=${builtins.toString ./.}"
+    ]
+    ++ pkgs.lib.concatMap (name: [
+      "--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}"
+      "--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}-libpath=${
+        pkgs.lib.getLib sharedLibDeps.${name}
+      }/lib"
+      "--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}-include=${
+        pkgs.lib.getInclude sharedLibDeps.${name}
+      }/include"
+    ]) (builtins.attrNames sharedLibDeps)
+  );
+}

tools/dep_updaters/update-nixpkgs-pin.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+set -ex
+# Shell script to update Nixpkgs pin in the source tree to the most recent
+# version on the unstable channel.
+
+BASE_DIR=$(cd "$(dirname "$0")/../.." && pwd)
+NIXPKGS_PIN_FILE="$BASE_DIR/tools/nix/pkgs.nix"
+
+NIXPKGS_REPO=$(grep 'repo =' "$NIXPKGS_PIN_FILE" | awk -F'"' '{ print $2 }')
+CURRENT_VERSION_SHA1=$(grep 'rev =' "$NIXPKGS_PIN_FILE" | awk -F'"' '{ print $2 }')
+CURRENT_VERSION=$(echo "$CURRENT_VERSION_SHA1" | head -c 7)
+
+NEW_UPSTREAM_SHA1=$(git ls-remote "$NIXPKGS_REPO.git" nixpkgs-unstable | awk '{print $1}')
+NEW_VERSION=$(echo "$NEW_UPSTREAM_SHA1" | head -c 7)
+
+
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
+compare_dependency_version "nixpkgs-unstable" "$NEW_VERSION" "$CURRENT_VERSION"
+
+CURRENT_TARBALL_HASH=$(grep 'sha256 =' "$NIXPKGS_PIN_FILE" | awk -F'"' '{ print $2 }')
+NEW_TARBALL_HASH=$(nix-prefetch-url --unpack "$NIXPKGS_REPO/archive/$NEW_UPSTREAM_SHA1.tar.gz")
+
+TMP_FILE=$(mktemp)
+sed "s/$CURRENT_VERSION_SHA1/$NEW_UPSTREAM_SHA1/;s/$CURRENT_TARBALL_HASH/$NEW_TARBALL_HASH/" "$NIXPKGS_PIN_FILE" > "$TMP_FILE"
+mv "$TMP_FILE" "$NIXPKGS_PIN_FILE"
+
+cat -<<EOF
+All done!
+
+Please git add and commit the new version:
+
+$ git add $NIXPKGS_PIN_FILE
+$ git commit -m 'tools: bump nixpkgs-unstable pin to $NEW_VERSION'
+EOF
+
+# The last line of the script should always print the new version,
+# as we need to add it to $GITHUB_ENV variable.
+echo "NEW_VERSION=$NEW_VERSION"

tools/nix/pkgs.nix

@@ -0,0 +1,10 @@
+arg:
+let
+  repo = "https://github.com/NixOS/nixpkgs";
+  rev = "ca77296380960cd497a765102eeb1356eb80fed0";
+  nixpkgs = import (builtins.fetchTarball {
+    url = "${repo}/archive/${rev}.tar.gz";
+    sha256 = "1airrw6l87iyny1a3mb29l28na4s4llifprlgpll2na461jd40iy";
+  }) arg;
+in
+nixpkgs