parallel compiler: `threads::spawn`ning loop not reproducible

[matthiaskrgr]

View all comments

I tried this code:

use std::thread;

fn _main() {
    let _t1 = thread::spawn(|| {
        for _ in 0..100 {
            println!("test");
        }
    });
}

compiled with RUSTFLAGS="" rustc a.rs -Zthreads=3 --crate-type lib -Clink-dead-code=true -Copt-level=0 -Ccodegen-units=1 ; md5sum liba.rlib

I expected to see this happen: explanation
same checksums all the time

Instead, this happened: explanation
different checksums:

ddeb820dec9e3451d3353aa1041e4ef3  liba.rlib
0604d96b3340f2b6e3bed389738515c9  liba.rlib
0604d96b3340f2b6e3bed389738515c9  liba.rlib
ddeb820dec9e3451d3353aa1041e4ef3  liba.rlib
0604d96b3340f2b6e3bed389738515c9  liba.rlib
0604d96b3340f2b6e3bed389738515c9  liba.rlib
3c3905784182c1b6958da68edba17c5d  liba.rlib
ddeb820dec9e3451d3353aa1041e4ef3  liba.rlib

Meta

rustc --version --verbose:

rustc 1.94.0-nightly (1107bbac4 2025-12-26)
binary: rustc
commit-hash: 1107bbac4b303d49c3e67a2ec62710902bf4b341
commit-date: 2025-12-26
host: x86_64-unknown-linux-gnu
release: 1.94.0-nightly
LLVM version: 21.1.8

[bjorn3]

I can reproduce this with rustc +nightly a.rs -Zthreads=3 --crate-type lib -Clink-dead-code=true; md5sum liba.rlib. It is the object file contained inside the rlib that changes. In particular the bitcode stored inside the object file.

diff -u <(objdump -D a.o) <(objdump -D b.o)
--- /proc/self/fd/11    2025-12-28 22:29:58.172799347 +0100
+++ /proc/self/fd/18    2025-12-28 22:29:58.172799347 +0100
@@ -1,5 +1,5 @@
 
-a.o:     file format elf64-x86-64
+b.o:     file format elf64-x86-64
 
 
 Disassembly of section .llvmbc:
@@ -3330,12 +3330,12 @@
     2169:      11 4c 90 8c             adc    %ecx,-0x74(%rax,%rdx,4)
     216d:      09 26                   or     %esp,(%rsi)
     216f:      47 c6 04 43 9a          rex.RXB movb $0x9a,(%r11,%r8,2)
-    2174:      94                      xchg   %eax,%esp
-    2175:      e2 2e                   loop   21a5 <.llvmbc+0x21a5>
-    2177:      2a 21                   sub    (%rcx),%ah
-    2179:      a0 ee 61 11 05 0b a2    movabs 0x4a08a20b051161ee,%al
-    2180:      08 4a 
-    2182:      80 00 23                addb   $0x23,(%rax)
+    2174:      14 a0                   adc    $0xa0,%al
+    2176:      b1 ed                   mov    $0xed,%cl
+    2178:      20 38                   and    %bh,(%rax)
+    217a:      3a ba 0d 05 0b a2       cmp    -0x5df4faf3(%rdx),%bh
+    2180:      08 4a 80                or     %cl,-0x80(%rdx)
+    2183:      00 23                   add    %ah,(%rbx)
     2185:      00 24 48                add    %ah,(%rax,%rcx,2)
     2188:      08 02                   or     %al,(%rdx)
     218a:      00 00                   add    %al,(%rax)
@@ -12669,19 +12669,23 @@
     88ad:      83 d9 01                sbb    $0x1,%ecx
     88b0:      ec                      in     (%dx),%al
     88b1:      3e 00 00                ds add %al,(%rax)
-    88b4:      8b 8a b7 e7 67 35       mov    0x3567e7b7(%rdx),%ecx
-    88ba:      0f 6a 5a 6f             punpckhdq 0x6f(%rdx),%mm3
-    88be:      5d                      pop    %rbp
-    88bf:      4f ec                   rex.WRXB in (%dx),%al
-    88c1:      c5 dd 4d                (bad)
-    88c4:      44 b4 7e                rex.R mov $0x7e,%spl
-    88c7:      e7 38                   out    %eax,$0x38
-    88c9:      50                      push   %rax
-    88ca:      41 19 57 41             sbb    %edx,0x41(%r15)
-    88ce:      01 00                   add    %eax,(%rax)
-    88d0:      65 0c 00                gs or  $0x0,%al
-    88d3:      00 d1                   add    %dl,%cl
-    88d5:      05 00 00 12 03          add    $0x3120000,%eax
+    88b4:      8b 0a                   mov    (%rdx),%ecx
+    88b6:      5a                      pop    %rdx
+    88b7:      7e e1                   jle    889a <.llvmbc+0x889a>
+    88b9:      7e 1d                   jle    88d8 <.llvmbc+0x88d8>
+    88bb:      d8 b9 77 d3 54 f8       fdivrs -0x7ab2c89(%rcx)
+    88c1:      d1 13                   rcll   $1,(%rbx)
+    88c3:      8c eb                   mov    %gs,%ebx
+    88c5:      60                      (bad)
+    88c6:      4f d9 70 ee             rex.WRXB fnstenv -0x12(%r8)
+    88ca:      54                      push   %rsp
+    88cb:      d6                      (bad)
+    88cc:      5d                      pop    %rbp
+    88cd:      d7                      xlat   %ds:(%rbx)
+    88ce:      06                      (bad)
+    88cf:      00 65 0c                add    %ah,0xc(%rbp)
+    88d2:      00 00                   add    %al,(%rax)
+    88d4:      d1 05 00 00 12 03       roll   $1,0x3120000(%rip)        # 31288da <_RINvNtNtCshSK8eIQvpwe_3std6thread9lifecycle15spawn_uncheckedNCNvCs75Gc0XuCLnc_3foo5__main0uEB10_+0x31288da>
     88da:      94                      xchg   %eax,%esp
     88db:      78 5e                   js     893b <.llvmbc+0x893b>
     88dd:      00 00                   add    %al,(%rax)

[zetanumbers]

What is "(bad)"? Is it a problem with the disassembler?

[bjorn3]

Objdump is trying to decode a non-code section (.llvmbc) as code. The hexdump here is the important part.

[bjorn3]

Looks like the only difference between both files is the value of !4. This is used on call void asm sideeffect "", "~{memory}"(), !srcloc !4 inside of two __rust_begin_short_backtrace definitions.

--- 59b9.ll     2026-02-12 13:23:56.629922184 +0100
+++ 71b9.ll     2026-02-12 13:24:02.729994411 +0100
@@ -1,4 +1,4 @@
-; ModuleID = '59b9.bc'
+; ModuleID = '71b9.bc'
 source_filename = "a.dca4690963c3a3d8-cgu.0"
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -5451,9 +5451,9 @@
 !1 = !{i32 2, !"RtLibUseGOT", i32 1}
 !2 = !{!"rustc version 1.95.0-nightly (7057231bd 2026-02-11)"}
 !3 = !{}
-!4 = !{i64 19650519060828136}
+!4 = !{i64 15809224864442949}
 
-^0 = module: (path: "59b9.bc", hash: (2359500773, 277542735, 440113564, 3477836827, 778154049))
+^0 = module: (path: "71b9.bc", hash: (2700998795, 3783365708, 1825895929, 222570575, 1475512440))
 ^1 = gv: (name: "_RNvNtNtNtCs11wwVK8ZSfd_3std2io5error14repr_bitpacked14kind_from_primCsiWtlKG0eWYm_1a", summaries: (function: (module: ^0, flags: (linkage: external, visibility: hidden, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0, importType: definition), insts: 173))) ; guid = 46444607873935881
 ^2 = gv: (name: "anon.cd76e29c2af7f4c2095a1f1bd415c011.59", summaries: (variable: (module: ^0, flags: (linkage: private, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0, importType: definition), varFlags: (readonly: 1, writeonly: 0, constant: 1)))) ; guid = 125661355751697100
 ^3 = gv: (name: "_RINvNtNtCs11wwVK8ZSfd_3std6thread9functions5spawnNCNvCsiWtlKG0eWYm_1a5__main0uEBP_", summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0, importType: definition), insts: 16, calls: ((callee: ^219), (callee: ^38)), refs: (^99, ^123, ^161)))) ; guid = 159319635833202446

; Function Attrs: noinline nonlazybind uwtable
define void @std::sys::backtrace::__rust_begin_short_backtrace::<std::thread::lifecycle::spawn_unchecked<a::_main::{closure#0}, ()>::{closure#1}::{closure#0}::{closure#0}, ()>(ptr align 8 %0) unnamed_addr #2 {
  call void @std::thread::lifecycle::spawn_unchecked::<a::_main::{closure#0}, ()>::{closure#1}::{closure#0}::{closure#0}(ptr align 8 %0) #26
  call void asm sideeffect "", "~{memory}"(), !srcloc !4
  ret void
}

; Function Attrs: noinline nonlazybind uwtable
define void @std::sys::backtrace::__rust_begin_short_backtrace::<a::_main::{closure#0}, ()>() unnamed_addr #2 {
  call void @a::_main::{closure#0}() #26
  call void asm sideeffect "", "~{memory}"(), !srcloc !4
  ret void
}

The !srcloc metadata is derived from the raw span locations in the source map at

rust/compiler/rustc_codegen_llvm/src/asm.rs

Lines 533 to 537 in 7ad4e69

	srcloc.extend(line_spans.iter().map(|span| {
	llvm::LLVMValueAsMetadata(
	bx.const_u64(u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32)),
	)
	}));

I guess there is some non-determinism somewhere inside rustc causing the order of files in the source map to change. Maybe we could use StableSourceFileId + offset inside the file rather than the raw span location? That would also allow inline asm errors to be shown with the original source location when LTO is used. Edit: StableSourceFileId hashes different between the local crate and dependencies as the StableCrateId is not available yet.

[bjorn3]

The cookie can only be 64bits, but StableSourceFileId is already 128bits and you need another 64bit for location+size within the file. And having a side table is likely to result in even more non-determinism as there has to be a single side table shared across codegen units for thin local LTO to work.

[xmakro]

I reproduced @bjorn3 diagnosis locally. A workaround is to not encode a cookie for imported-file spans and change rustc_codegen_llvm/src/asm.rs#L537 to:

if source_map.is_imported(*span) { 0 } else { u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32) }

This makes the output deterministic across many runs. It matches what report_inline_asm already does under LTO. But I'm not sure if suppressing the foreign cookie is acceptable. Otherwise we need to make the import order deterministic.

[bjorn3]

That would workaround the current non-determinism issue, but that is still vulnerable to non-determinism if we start doing parallel macro expansion in the future as then non-imported source files will be in non-deterministic order too. We have to stop encoding the raw span location directly in the LLVM IR somehow.

[susitsm]

@rustbot claim
I am thinking of adding a StableBytePos which is BytePos, but remapped to a stable sorted SourceMap order. Something like

 bx.const_u64(u64::from(span.lo().to_stable_sorted(tcx).to_u32()) | (u64::from(span.hi().to_stable_sorted(tcx).to_u32()) << 32)) 

then change the decoding to match it.

[bjorn3]

That would break horribly if we ever add a new source file during codegen as that would shift all spans around. I was hoping we could change LLVM to accept arbitrary sized cookies and then encode the start and end of spans as stable_source_file_id + offset from start of source file. This would take a 320 bit integer.

[susitsm]

It would, but it is possible to lock the SourceMap during codegen and preload everything.
I don't know how long it would take for such a patch to be accepted into LLVM. It would also fix the full LTO problem, so it is better from a technical point of view.
Question is what is the priority on this issue? Get it fixed fast, get it fixed with the next LLVM release or add a custom patch to the rust fork?