chore: Upgrade RTS Dependencies (#4677)

Upgrading Motoko RTS dependencies: * LLVM 18.1.8 * Rust 1.82.0 Special aspects: * Support new `dylink.0` instead of `dylink`. * Support `GOT.mem` for RTS in the Motoko linker. * Adjusting the RTS code in line with the latest Rust features/rules. * No longer needing `emscripten`: Use custom Rust targets . * Get rid of `emscripten` by using custom Rust targets for generating PIC-relocatable Wasm. * Get rid of `musl` by using Rust builtins instead. * Float formatting changes due to removal of `musl`: The output format of `NaN` and `infinity` has changed. Moreover, one particular output format (hexadecimal float) is no longer supported. Updating the documentation and base library is needed. * Optimization: Use bulk-memory operations because memcpy of Rust is slower than `musl`. Note: * If your `nix-shell` crashes with a segmentation fault with this PR, you need to upgrade `nix`. For this, you need to manually uninstall and re-install `nix`.
dfinity · Nov 29, 2024 · bdb1d24 · bdb1d24
1 parent aebc110
commit bdb1d24
Show file tree

Hide file tree

Showing 62 changed files with 699 additions and 847 deletions.
diff --git a/.github/workflows/niv-updater-rare.yml b/.github/workflows/niv-updater-rare.yml
@@ -23,7 +23,7 @@ jobs:
         uses: knl/niv-updater-action@v15
         with:
           # might be too noisy
-          whitelist: 'ic-ref,musl-wasi,ic-wasm'
+          whitelist: 'ic-ref,ic-wasm'
           labels: |
             automerge-squash
           keep_updating: true

diff --git a/.github/workflows/niv-updater-trial.yml b/.github/workflows/niv-updater-trial.yml
@@ -28,7 +28,7 @@ jobs:
       - name: niv-updater-action
         uses: knl/niv-updater-action@v15
         with:
-          whitelist: 'nixpkgs,musl-wasi,ic'
+          whitelist: 'nixpkgs,ic'
           labels: |
             autoclose
           keep_updating: true

diff --git a/.github/workflows/niv-updater.yml b/.github/workflows/niv-updater.yml
@@ -23,7 +23,7 @@ jobs:
         uses: knl/niv-updater-action@v15
         with:
           # might be too noisy
-          blacklist: 'nixpkgs,ic-ref,musl-wasi,ic'
+          blacklist: 'nixpkgs,ic-ref,ic'
           labels: |
             automerge-squash
           keep_updating: true

diff --git a/Changelog.md b/Changelog.md
@@ -1,5 +1,11 @@
 # Motoko compiler changelog
 
+## 0.13.4 (2024-11-29)
+
+* motoko (`moc`)
+
+  * refactoring: Updating and simplifying the runtime system dependencies (#4677).
+
 ## 0.13.3 (2024-11-13)
 
 * motoko (`moc`)

diff --git a/default.nix b/default.nix
@@ -35,35 +35,31 @@ in
 
 let
   rtsBuildInputs = with nixpkgs; [
-    # pulls in clang (wrapped) and clang-13 (unwrapped)
-    llvmPackages_13.clang
-    # pulls in wasm-ld
-    llvmPackages_13.lld
-    llvmPackages_13.bintools
+    llvmPackages_18.clang
+    llvmPackages_18.bintools
     rustc-nightly
     cargo-nightly
     wasmtime
     rust-bindgen
     python3
-    emscripten
   ] ++ pkgs.lib.optional pkgs.stdenv.isDarwin [
     libiconv
   ];
 
   llvmEnv = ''
     # When compiling to wasm, we want to have more control over the flags,
     # so we do not use the nix-provided wrapper in clang
-    export WASM_CLANG="clang-13"
+    export WASM_CLANG="clang-18"
     export WASM_LD=wasm-ld
     # because we use the unwrapped clang, we have to pass in some flags/paths
     # that otherwise the wrapped clang would take care for us
-    export WASM_CLANG_LIB="${nixpkgs.llvmPackages_13.clang-unwrapped.lib}"
+    export WASM_CLANG_LIB="${nixpkgs.llvmPackages_18.clang-unwrapped.lib}"
 
     # When compiling natively, we want to use `clang` (which is a nixpkgs
     # provided wrapper that sets various include paths etc).
     # But for some reason it does not handle building for Wasm well, so
-    # there we use plain clang-13. There is no stdlib there anyways.
-    export CLANG="${nixpkgs.clang_13}/bin/clang"
+    # there we use plain clang-18. There is no stdlib there anyways.
+    export CLANG="${nixpkgs.clang_18}/bin/clang"
   '';
 in
 
@@ -153,15 +149,6 @@ let ocaml_exe = name: bin: rts:
         $out/bin/* --help >/dev/null
       '';
     };
-
-  musl-wasi-sysroot = stdenv.mkDerivation {
-    name = "musl-wasi-sysroot";
-    src = nixpkgs.sources.musl-wasi;
-    phases = [ "unpackPhase" "installPhase" ];
-    installPhase = ''
-      make SYSROOT="$out" include_dirs
-    '';
-  };
 in
 
 rec {
@@ -178,7 +165,7 @@ rec {
       vendorRustStdDeps = "${cargoVendorTools}/bin/vendor-rust-std-deps";
 
       # SHA256 of Rust std deps
-      rustStdDepsHash = "sha256-A3WPIx+weu4wIYV7cweGkRxYGAPt7srxBAtMEyPOkhI=";
+      rustStdDepsHash = "sha256-U4BTr1CzFuOMdyLuhw5ry3/u8bkRiPmnMr4pLo3IdOQ=";
 
       # Vendor directory for Rust std deps
       rustStdDeps = nixpkgs.stdenvNoCC.mkDerivation {
@@ -255,8 +242,6 @@ rec {
         
         ${llvmEnv}
         export TOMMATHSRC=${nixpkgs.sources.libtommath}
-        export MUSLSRC=${nixpkgs.sources.musl-wasi}/libc-top-half/musl
-        export MUSL_WASI_SYSROOT=${musl-wasi-sysroot}
       '';
 
       doCheck = true;
@@ -343,7 +328,7 @@ rec {
 
     # extra deps for test/ld
     ldTestDeps =
-      with nixpkgs; [ llvmPackages_13.bintools llvmPackages_13.clang ];
+      with nixpkgs; [ llvmPackages_18.lld llvmPackages_18.clang ];
 
     testDerivation = args:
       stdenv.mkDerivation (testDerivationArgs // args);
@@ -865,8 +850,6 @@ EOF
     '';
     ESM=nixpkgs.sources.esm;
     TOMMATHSRC = nixpkgs.sources.libtommath;
-    MUSLSRC = "${nixpkgs.sources.musl-wasi}/libc-top-half/musl";
-    MUSL_WASI_SYSROOT = musl-wasi-sysroot;
     LOCALE_ARCHIVE = nixpkgs.lib.optionalString stdenv.isLinux "${nixpkgs.glibcLocales}/lib/locale/locale-archive";
     MOTOKO_BASE = base-src;
     CANDID_TESTS = "${nixpkgs.sources.candid}/test";

diff --git a/nix/default.nix b/nix/default.nix
@@ -91,13 +91,11 @@ let
 
         # Rust nightly
         (self: super: let
-          rust-channel = self.moz_overlay.rustChannelOf { date = "2023-04-21"; channel = "nightly"; };
+          rust-channel = self.moz_overlay.rustChannelOf { date = "2024-07-28"; channel = "nightly"; };
         in rec {
           rustc-nightly = rust-channel.rust.override {
             targets = [
-               "wasm32-unknown-emscripten"
                "wasm32-wasi"
-               "i686-unknown-linux-gnu"
             ];
             extensions = ["rust-src"];
           };

diff --git a/nix/drun.nix b/nix/drun.nix
@@ -72,17 +72,17 @@ EOF
 
       buildInputs = with pkgs; [
         openssl
-        llvm_13
-        llvmPackages_13.libclang
+        llvm_18
+        llvmPackages_18.libclang
         lmdb
         libunwind
         libiconv
       ] ++ pkgs.lib.optional pkgs.stdenv.isDarwin
         pkgs.darwin.apple_sdk.frameworks.Security;
 
       # needed for bindgen
-      LIBCLANG_PATH = "${pkgs.llvmPackages_13.libclang.lib}/lib";
-      CLANG_PATH = "${pkgs.llvmPackages_13.clang}/bin/clang";
+      LIBCLANG_PATH = "${pkgs.llvmPackages_18.libclang.lib}/lib";
+      CLANG_PATH = "${pkgs.llvmPackages_18.clang}/bin/clang";
 
       # needed for ic-protobuf
       PROTOC="${pkgs.protobuf}/bin/protoc";

diff --git a/nix/sources.json b/nix/sources.json
@@ -75,10 +75,10 @@
         "homepage": "",
         "owner": "dfinity",
         "repo": "motoko-base",
-        "rev": "cf840985f48736d22e52db26147c9f9a2dcd19b1",
-        "sha256": "11y0nql11w9x1crg6szy0w1pa42nixld91gvn2sbiqxgpbrw02r8",
+        "rev": "2d4135ff07470802b39e94017ebb7e46e671c59b",
+        "sha256": "0ffxacbdakkhbrnchdxrlz62c8yd1jfy77krsf0vd1ihbf48svdr",
         "type": "tarball",
-        "url": "https://github.com/dfinity/motoko-base/archive/cf840985f48736d22e52db26147c9f9a2dcd19b1.tar.gz",
+        "url": "https://github.com/dfinity/motoko-base/archive/2d4135ff07470802b39e94017ebb7e46e671c59b.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "motoko-matchers": {
@@ -94,30 +94,16 @@
         "url": "https://github.com/kritzcreek/motoko-matchers/archive/cb838c192df3328ff9ae172e2dc7338cf55e74bf.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
-    "musl-wasi": {
-        "branch": "main",
-        "builtin": false,
-        "description": "WASI libc implementation for WebAssembly",
-        "homepage": "https://wasi.dev",
-        "owner": "WebAssembly",
-        "repo": "wasi-libc",
-        "rev": "c5264e2bbe532994d06b039005f2af91bedcc1a6",
-        "sha256": "1skw2jqzaidr2zbzbjw32b36xvqniwf4if8cr4kbpp8vjvh7j2lr",
-        "type": "tarball",
-        "url": "https://github.com/WebAssembly/wasi-libc/archive/c5264e2bbe532994d06b039005f2af91bedcc1a6.tar.gz",
-        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
-    },
     "nixpkgs": {
         "branch": "release-24.05",
-        "builtin": true,
-        "description": "Nixpkgs/NixOS branches that track the Nixpkgs/NixOS channels",
-        "homepage": null,
+        "description": "Nix Packages collection & NixOS",
+        "homepage": "",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "bb8c2cf7ea0dd2e18a52746b2c3a5b0c73b93c22",
-        "sha256": "03kvrs5ai4y09hcjkbzwlxx0gv5sggqi0m82cxdqikkyriwlg97b",
+        "rev": "404f983f8c1da3ed403c95df9bd18d7c3b23399d",
+        "sha256": "025alb35qf1iwmz45615ahp400q54582kqzc0238a51ai2w88pxc",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/bb8c2cf7ea0dd2e18a52746b2c3a5b0c73b93c22.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs/archive/404f983f8c1da3ed403c95df9bd18d7c3b23399d.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "nixpkgs-mozilla": {