Non-deterministic fatal go runtime crashes on macOS

[tsandall]

Hi there!

I'm running into some non-deterministic go runtime crashes on macOS w/ v0.23.0. Here's an example:

package main

import (
	"fmt"

	"github.com/bytecodealliance/wasmtime-go"

	_ "github.com/bytecodealliance/wasmtime-go/build/include"        // to include the C headers.
	_ "github.com/bytecodealliance/wasmtime-go/build/linux-x86_64"   // to include the static lib for linking.
	_ "github.com/bytecodealliance/wasmtime-go/build/macos-x86_64"   // to include the static lib for linking.
	_ "github.com/bytecodealliance/wasmtime-go/build/windows-x86_64" // to include the static lib for linking.
)

func main() {

	bs, err := wasmtime.Wat2Wasm(`
	(module
		(func (export "f1")
			(unreachable)))
	`)
	check(err)

	for i := 0; ; i++ {
		fmt.Println("i:", i)
		store := wasmtime.NewStore(wasmtime.NewEngine())

		module, err := wasmtime.NewModule(store.Engine, bs)
		check(err)

		instance, err := wasmtime.NewInstance(store, module, nil)
		check(err)

		_, err = instance.GetExport("f1").Func().Call()
		if err == nil {
			panic("expected error")
		}
	}

}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

If you run this program on macOS, it should crash eventually (the last couple runs on my laptop took about 150 iterations, but it's not consistent).

It seems like something nasty is happening in the runtime when the unreachable instruction is executed.

Here's the output from the last run:

$ go run -mod=vendor main.go
i: 0
i: 1
<--------------8< snip snip 8<----------------->
i: 296
signal 16 received but handler not on signal stack
fatal error: non-Go code set up signal handler without SA_ONSTACK flag

runtime stack:
runtime: unexpected return pc for runtime.sigtramp called from 0xc000127c28
stack: frame={sp:0xc0001277b0, fp:0xc0001277c0} stack=[0xc00011f6e0,0xc000127ae0)
000000c0001276b0:  000000c0001276d0  0000000004049665 <runtime.sigNotOnStack+133>
000000c0001276c0:  0000000004c15568  0000000000000039
000000c0001276d0:  000000c000127728  00000000040483db <runtime.adjustSignalStack+411>
000000c0001276e0:  0000000000000010  000000c0001276f8
000000c0001276f0:  000000c000127738  0000000008c22000
000000c000127700:  0000000000010000  0000000000000001
000000c000127710:  000000c00010a330  0000000004c15c30
000000c000127720:  0000000000000000  000000c0001277a0
000000c000127730:  0000000004048131 <runtime.sigtrampgo+369>  000000c000000010
000000c000127740:  0000000004e8aea0  000000c000127760
000000c000127750:  0000000000000000  0000000000000000
000000c000127760:  0000000000000000  0000000000000000
000000c000127770:  0000000000000000  0000000000000000
000000c000127780:  0000000000000000  000000c000000180
000000c000127790:  000000c000127c28  000000c000127c90
000000c0001277a0:  000000c0001277f0  0000000004067db3 <runtime.sigtramp+51>
000000c0001277b0: <0000000000000010 !000000c000127c28
000000c0001277c0: >000000c000127c90  000000c000127c90
000000c0001277d0:  d1560bb8e9a711de  000000000000013e
000000c0001277e0:  000000000000013d  0000000000000200
000000c0001277f0:  000000c000127800  00007fff6c4b05fd
000000c000127800:  000000c000127dd8  0000000000000000
000000c000127810:  0000000000000000  0000000000000000
000000c000127820:  00000000000000de  0000000037140000
000000c000127830:  000000000600a318  0000000004e8aea0
000000c000127840:  000000c000000180  0000000000000008
000000c000127850:  0000000006000108  0000000000000000
000000c000127860:  000000c000127dd8  000000c000127d48
000000c000127870:  0000000004beb8e0  0000000000003001
000000c000127880:  0000000000000018  0000000000000000
000000c000127890:  ffffffffffffffff  000000000000013e
000000c0001278a0:  000000000000013d  0000000000000200
000000c0001278b0:  000000000400ea3b <runtime.mallocgc+667>  0000000000000202
runtime.throw(0x4c15568, 0x39)
	/usr/local/Cellar/go/1.15.8/libexec/src/runtime/panic.go:1116 +0x72
runtime.sigNotOnStack(0x10)
	/usr/local/Cellar/go/1.15.8/libexec/src/runtime/signal_unix.go:926 +0x85
runtime.adjustSignalStack(0xc000000010, 0x4e8aea0, 0xc000127760, 0x0)
	/usr/local/Cellar/go/1.15.8/libexec/src/runtime/signal_unix.go:516 +0x19b
runtime.sigtrampgo(0x10, 0xc000127c28, 0xc000127c90)
	/usr/local/Cellar/go/1.15.8/libexec/src/runtime/signal_unix.go:461 +0x171
runtime: unexpected return pc for runtime.sigtramp called from 0xc000127c28
stack: frame={sp:0xc0001277b0, fp:0xc0001277c0} stack=[0xc00011f6e0,0xc000127ae0)
000000c0001276b0:  000000c0001276d0  0000000004049665 <runtime.sigNotOnStack+133>
000000c0001276c0:  0000000004c15568  0000000000000039
000000c0001276d0:  000000c000127728  00000000040483db <runtime.adjustSignalStack+411>
000000c0001276e0:  0000000000000010  000000c0001276f8
000000c0001276f0:  000000c000127738  0000000008c22000
000000c000127700:  0000000000010000  0000000000000001
000000c000127710:  000000c00010a330  0000000004c15c30
000000c000127720:  0000000000000000  000000c0001277a0
000000c000127730:  0000000004048131 <runtime.sigtrampgo+369>  000000c000000010
000000c000127740:  0000000004e8aea0  000000c000127760
000000c000127750:  0000000000000000  0000000000000000
000000c000127760:  0000000000000000  0000000000000000
000000c000127770:  0000000000000000  0000000000000000
000000c000127780:  0000000000000000  000000c000000180
000000c000127790:  000000c000127c28  000000c000127c90
000000c0001277a0:  000000c0001277f0  0000000004067db3 <runtime.sigtramp+51>
000000c0001277b0: <0000000000000010 !000000c000127c28
000000c0001277c0: >000000c000127c90  000000c000127c90
000000c0001277d0:  d1560bb8e9a711de  000000000000013e
000000c0001277e0:  000000000000013d  0000000000000200
000000c0001277f0:  000000c000127800  00007fff6c4b05fd
000000c000127800:  000000c000127dd8  0000000000000000
000000c000127810:  0000000000000000  0000000000000000
000000c000127820:  00000000000000de  0000000037140000
000000c000127830:  000000000600a318  0000000004e8aea0
000000c000127840:  000000c000000180  0000000000000008
000000c000127850:  0000000006000108  0000000000000000
000000c000127860:  000000c000127dd8  000000c000127d48
000000c000127870:  0000000004beb8e0  0000000000003001
000000c000127880:  0000000000000018  0000000000000000
000000c000127890:  ffffffffffffffff  000000000000013e
000000c0001278a0:  000000000000013d  0000000000000200
000000c0001278b0:  000000000400ea3b <runtime.mallocgc+667>  0000000000000202
runtime.sigtramp(0xc000127c90, 0xc000127c90, 0xd1560bb8e9a711de, 0x13e, 0x13d, 0x200, 0xc000127800, 0x7fff6c4b05fd, 0xc000127dd8, 0x0, ...)
	/usr/local/Cellar/go/1.15.8/libexec/src/runtime/sys_darwin_amd64.s:229 +0x33

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
	/usr/local/Cellar/go/1.15.8/libexec/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc000066fe8 sp=0xc000066fe0 pc=0x4066481

goroutine 1 [running]:
	goroutine running on other thread; stack unavailable
exit status 2

I'm running go1.15.8 after I found the other issues related to older versions (<1.14.11, I think).

One other data point/question...

I've found that if a Go-defined function is called and panics before the unreachable instruction is executed, I still run into the same crash. For example, here's the WAT:

		(import "" "f2" (func $f2))
		(func (export "f1")
			(call $f2)
			(unreachable)))

In this case, the Go-defined function "f2" just calls panic().

I'm on macOS 10.15.7 in case that helps.

Would rewriting my Go-defined functions to use Traps instead of panicking help at all?

[alexcrichton]

Thanks for the report! I've also been seeing this on CI for some time now, I just unfortunately haven't had the time to investigate yet :(

[alexcrichton]

After digging a bit further I opened up golang/go#44501 so I'll watch that and see how it goes.

[alexcrichton]

Ok after some discussion on the linked issue I think this should be fixed by bytecodealliance/wasmtime#2676

[srenatus]

I haven't seen this in a while now, and we've reintroduced our unreachable instructions. This can be closed as far as I can see. 👏

[alexcrichton]

Ok!

[srenatus]

😢 I'm afraid I was wrong. Running @tsandall's example code still reliably ends with a SIGILL:

[....]

i: 3193
SIGILL: illegal instruction
PC=0x59b70004 m=7 sigcode=1
instruction bytes: 0xf 0xb 0x55 0x48 0x89 0xe5 0xff 0xd2 0x48 0x89 0xec 0x5d 0xc3 0x0 0x0 0x0

goroutine 0 [idle]:
runtime: unknown pc 0x59b70004
stack: frame={sp:0x700004fe6a10, fp:0x0} stack=[0x700004f67350,0x700004fe6f50)
0000700004fe6910:  0000000000000000  0000000000000000
0000700004fe6920:  0000000000000000  0000000000000000
0000700004fe6930:  0000000000000000  0000000000000000
0000700004fe6940:  0000000000000000  0000000000000000
0000700004fe6950:  0000000000000000  0000000000000000
0000700004fe6960:  0000000000000000  0000000000000000
0000700004fe6970:  0000000000000000  0000000000000000
0000700004fe6980:  0000000000000000  0000000000000000
0000700004fe6990:  0000000000000000  0000000000000000
0000700004fe69a0:  0000000000000000  0000000000000000
0000700004fe69b0:  0000000000000000  0000000000000000
0000700004fe69c0:  0000000004067f20 <runtime.madvise_trampoline+0>  0000700004fe6a18
0000700004fe69d0:  0000000000000000  000000c00050e000
0000700004fe69e0:  0000700004fe6a08  00000000040528b1 <runtime.madvise+49>
0000700004fe69f0:  0000000004067f20 <runtime.madvise_trampoline+0>  0000700004fe6a18
0000700004fe6a00:  0000000000000000  0000700004fe6a88
0000700004fe6a10: <0000700004fe6a20  0000000059b7000c
0000700004fe6a20:  0000700004fe6b00  00000000047b6d8c
0000700004fe6a30:  0000000000000000  2fdaf3cff5245db6
0000700004fe6a40:  2fdaf3cff5245c86  0000700004fe6b50
0000700004fe6a50:  0000700004fe6a30  0000700004fe6cd0
0000700004fe6a60:  00000000043c0a90  2fda83cff5a15bc8
0000700004fe6a70:  000000000401628f <runtime.heapBits.forwardOrBoundary+111>  0000037f00001fa0
0000700004fe6a80:  0000000000000000  0000000005d1ffff
0000700004fe6a90:  0000000000000400  0000000000000000
0000700004fe6aa0:  000000c000482f00  000000c00048b5b0
0000700004fe6ab0:  0000000000000000  0000700004fe6b18
0000700004fe6ac0:  000000004f98df60  0000700004fe6cd0
0000700004fe6ad0:  1edaebe6e0380017  0000700004fe6cd0
0000700004fe6ae0:  0000000000000008  0000700004fe6da8
0000700004fe6af0:  0000700004fe6d20  0000700004fe6b50
0000700004fe6b00:  0000700004fe6bc0  00000000043c0b4f
runtime: unknown pc 0x59b70004
stack: frame={sp:0x700004fe6a10, fp:0x0} stack=[0x700004f67350,0x700004fe6f50)
0000700004fe6910:  0000000000000000  0000000000000000
0000700004fe6920:  0000000000000000  0000000000000000
0000700004fe6930:  0000000000000000  0000000000000000
0000700004fe6940:  0000000000000000  0000000000000000
0000700004fe6950:  0000000000000000  0000000000000000
0000700004fe6960:  0000000000000000  0000000000000000
0000700004fe6970:  0000000000000000  0000000000000000
0000700004fe6980:  0000000000000000  0000000000000000
0000700004fe6990:  0000000000000000  0000000000000000
0000700004fe69a0:  0000000000000000  0000000000000000
0000700004fe69b0:  0000000000000000  0000000000000000
0000700004fe69c0:  0000000004067f20 <runtime.madvise_trampoline+0>  0000700004fe6a18
0000700004fe69d0:  0000000000000000  000000c00050e000
0000700004fe69e0:  0000700004fe6a08  00000000040528b1 <runtime.madvise+49>
0000700004fe69f0:  0000000004067f20 <runtime.madvise_trampoline+0>  0000700004fe6a18
0000700004fe6a00:  0000000000000000  0000700004fe6a88
0000700004fe6a10: <0000700004fe6a20  0000000059b7000c
0000700004fe6a20:  0000700004fe6b00  00000000047b6d8c
0000700004fe6a30:  0000000000000000  2fdaf3cff5245db6
0000700004fe6a40:  2fdaf3cff5245c86  0000700004fe6b50
0000700004fe6a50:  0000700004fe6a30  0000700004fe6cd0
0000700004fe6a60:  00000000043c0a90  2fda83cff5a15bc8
0000700004fe6a70:  000000000401628f <runtime.heapBits.forwardOrBoundary+111>  0000037f00001fa0
0000700004fe6a80:  0000000000000000  0000000005d1ffff
0000700004fe6a90:  0000000000000400  0000000000000000
0000700004fe6aa0:  000000c000482f00  000000c00048b5b0
0000700004fe6ab0:  0000000000000000  0000700004fe6b18
0000700004fe6ac0:  000000004f98df60  0000700004fe6cd0
0000700004fe6ad0:  1edaebe6e0380017  0000700004fe6cd0
0000700004fe6ae0:  0000000000000008  0000700004fe6da8
0000700004fe6af0:  0000700004fe6d20  0000700004fe6b50
0000700004fe6b00:  0000700004fe6bc0  00000000043c0b4f

goroutine 1 [syscall]:
runtime.cgocall(0x40b87f0, 0xc000065d00, 0xc000482f00)
        /nix/store/9xx9iq19j8wm4vaj8042jkzyx06ypa5z-go-1.15.4/share/go/src/runtime/cgocall.go:133 +0x5b fp=0xc000065cd0 sp=0xc000065c98 pc=0x400547b
github.com/bytecodealliance/wasmtime-go._Cfunc_go_wasmtime_func_call(0x4f98e920, 0xc000482ef0, 0xc000482f00, 0xc00048b5b0, 0x0)
        _cgo_gotypes.go:718 +0x4e fp=0xc000065d00 sp=0xc000065cd0 pc=0x40ad70e
github.com/bytecodealliance/wasmtime-go.(*Func).Call.func3.1(0xc00048e6e0, 0xc000482ef0, 0xc000482f00, 0xc00048b5b0, 0x4d36201)
        /Users/stephan/Sources/open-policy-agent/opa/vendor/github.com/bytecodealliance/wasmtime-go/func.go:505 +0x117 fp=0xc000065d40 sp=0xc000065d00 pc=0x40b58d7
github.com/bytecodealliance/wasmtime-go.(*Func).Call.func3(0xc00048b5b0)
        /Users/stephan/Sources/open-policy-agent/opa/vendor/github.com/bytecodealliance/wasmtime-go/func.go:505 +0x4f fp=0xc000065d80 sp=0xc000065d40 pc=0x40b594f
github.com/bytecodealliance/wasmtime-go.enterWasm(0xc00048c7e0, 0xc000065eb0, 0x0)
        /Users/stephan/Sources/open-policy-agent/opa/vendor/github.com/bytecodealliance/wasmtime-go/func.go:589 +0xf0 fp=0xc000065de0 sp=0xc000065d80 pc=0x40b1d70
github.com/bytecodealliance/wasmtime-go.(*Func).Call(0xc00048e6e0, 0x0, 0x0, 0x0, 0xc000490088, 0xc0004810e0, 0x0, 0x0)
        /Users/stephan/Sources/open-policy-agent/opa/vendor/github.com/bytecodealliance/wasmtime-go/func.go:500 +0x2be fp=0xc000065ee8 sp=0xc000065de0 pc=0x40b167e
main.main()
        /Users/stephan/Sources/open-policy-agent/opa/scratch/issue-3225/main.go:33 +0x225 fp=0xc000065f88 sp=0xc000065ee8 pc=0x40b7b65
runtime.main()
        /nix/store/9xx9iq19j8wm4vaj8042jkzyx06ypa5z-go-1.15.4/share/go/src/runtime/proc.go:204 +0x209 fp=0xc000065fe0 sp=0xc000065f88 pc=0x4037ba9
runtime.goexit()
        /nix/store/9xx9iq19j8wm4vaj8042jkzyx06ypa5z-go-1.15.4/share/go/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc000065fe8 sp=0xc000065fe0 pc=0x40667e1

rax    0x4f98f8b0
rbx    0x0
rcx    0x8
rdx    0x59b70000
rdi    0x4f98f8b0
rsi    0x0
rbp    0x700004fe6a10
rsp    0x700004fe6a10
r8     0x59b70006
r9     0xc00048b5b0
r10    0xc000482f00
r11    0x0
r12    0x700004fe6b50
r13    0x700004fe6a30
r14    0x700004fe6cd0
r15    0x43c0a90
rip    0x59b70004
rflags 0x10246
cs     0x2b
fs     0x0
gs     0x0
exit status 2

[srenatus]

I've been stumbling upon this in some work related to use interrupting the VM. It turns out that those suffer a similar fate to what Torin initially reported here, presumably interrupt traps and unreachable traps have something in common.

This is my main.go,

package main

import (
	"log"
	"time"

	"github.com/bytecodealliance/wasmtime-go"

	_ "github.com/bytecodealliance/wasmtime-go/build/include"        // to include the C headers.
	_ "github.com/bytecodealliance/wasmtime-go/build/linux-x86_64"   // to include the static lib for linking.
	_ "github.com/bytecodealliance/wasmtime-go/build/macos-x86_64"   // to include the static lib for linking.
	_ "github.com/bytecodealliance/wasmtime-go/build/windows-x86_64" // to include the static lib for linking.
)

func main() {
	bs, err := wasmtime.Wat2Wasm(`
	  (func (export "loop")
	    (loop br 0))
	`)
	check(err)

	for i := 0; ; i++ {
		config := wasmtime.NewConfig()
		config.SetInterruptable(true)
		store := wasmtime.NewStore(wasmtime.NewEngineWithConfig(config))

		module, err := wasmtime.NewModule(store.Engine, bs)
		check(err)

		instance, err := wasmtime.NewInstance(store, module, nil)
		check(err)

		errc := make(chan error)
		go func() {
			_, err = instance.GetExport("loop").Func().Call()
			errc <- err
		}()
		time.Sleep(100*time.Millisecond)

		handle, err := store.InterruptHandle()
		check(err)

		handle.Interrupt()

		err = <-errc
		if err == nil {
			panic("expected error")
		}
		if t, ok := err.(*wasmtime.Trap); ok {
			log.Printf("%d: trap: %s", i, t.Message())
		}
	}

}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

and this is a run that ends in a 💥:

[...]
2021/04/20 15:47:03 2256: trap: wasm trap: interrupt
wasm backtrace:
    0:   0x21 - <unknown>!<wasm function 0>
SIGILL: illegal instruction
PC=0x4a10001a m=7 sigcode=1
instruction bytes: 0xf 0xb 0x55 0x48 0x89 0xe5 0xff 0xd2 0x48 0x89 0xec 0x5d 0xc3 0x0 0x0 0x0

goroutine 0 [idle]:
runtime: unknown pc 0x4a10001a
stack: frame={sp:0x7000012c8a10, fp:0x0} stack=[0x700001249350,0x7000012c8f50)
00007000012c8910:  0000000000000000  0000000000000000
00007000012c8920:  0000000000000000  0000000000000000
00007000012c8930:  0000000000000000  0000000000000000
00007000012c8940:  0000000000000000  0000000000000000
00007000012c8950:  0000000000000000  0000000000000000
00007000012c8960:  0000000000000000  0000000000000000
00007000012c8970:  0000000000000000  0000000000000000
00007000012c8980:  0000000000000000  0000000000000000
00007000012c8990:  0000000000000000  0000000000000000
00007000012c89a0:  0000000000000000  0000000000000000
00007000012c89b0:  0000000000000000  0000000000000000
00007000012c89c0:  0000000000000000  0000000000000000
00007000012c89d0:  0000000000000000  00007000012c89f8
00007000012c89e0:  0000000000000000  000000000402dd5d <runtime.(*pageCache).allocN+61>
00007000012c89f0:  00000000040674c1 <runtime.goexit+1>  000000000004458f
00007000012c8a00:  00000000040bc513 <main.main.func1+147>  0000005800073ed6
00007000012c8a10: <00007000012c8a20  000000004a100022
00007000012c8a20:  00007000012c8b00  00000000047bb72c
00007000012c8a30:  0000000000000000  1fe0f637d94e91db
00007000012c8a40:  1fe0f637d94e90eb  00007000012c8b50
00007000012c8a50:  00007000012c8a30  00007000012c8cd0
00007000012c8a60:  00000000043c5430  1fe08637dc19adc5
00007000012c8a70:  00000000040b53eb <github.com/bytecodealliance/wasmtime-go.(*Func).Type+43>  0000037f00001fa0
00007000012c8a80:  0000000000000000  0000000000000000
00007000012c8a90:  0000000004019f5e <runtime.(*mcentral).grow+318>  0000000000000000
00007000012c8aa0:  000000c00061a010  000000c000610018
00007000012c8ab0:  0000000000000000  00007000012c8b18
00007000012c8ac0:  0000000047c4c130  00007000012c8cd0
00007000012c8ad0:  4d08e922884200c0  00007000012c8cd0
00007000012c8ae0:  0000000000000008  00007000012c8da8
00007000012c8af0:  00007000012c8d20  00007000012c8b50
00007000012c8b00:  00007000012c8bc0  00000000043c54ef
runtime: unknown pc 0x4a10001a
stack: frame={sp:0x7000012c8a10, fp:0x0} stack=[0x700001249350,0x7000012c8f50)
00007000012c8910:  0000000000000000  0000000000000000
00007000012c8920:  0000000000000000  0000000000000000
00007000012c8930:  0000000000000000  0000000000000000
00007000012c8940:  0000000000000000  0000000000000000
00007000012c8950:  0000000000000000  0000000000000000
00007000012c8960:  0000000000000000  0000000000000000
00007000012c8970:  0000000000000000  0000000000000000
00007000012c8980:  0000000000000000  0000000000000000
00007000012c8990:  0000000000000000  0000000000000000
00007000012c89a0:  0000000000000000  0000000000000000
00007000012c89b0:  0000000000000000  0000000000000000
00007000012c89c0:  0000000000000000  0000000000000000
00007000012c89d0:  0000000000000000  00007000012c89f8
00007000012c89e0:  0000000000000000  000000000402dd5d <runtime.(*pageCache).allocN+61>
00007000012c89f0:  00000000040674c1 <runtime.goexit+1>  000000000004458f
00007000012c8a00:  00000000040bc513 <main.main.func1+147>  0000005800073ed6
00007000012c8a10: <00007000012c8a20  000000004a100022
00007000012c8a20:  00007000012c8b00  00000000047bb72c
00007000012c8a30:  0000000000000000  1fe0f637d94e91db
00007000012c8a40:  1fe0f637d94e90eb  00007000012c8b50
00007000012c8a50:  00007000012c8a30  00007000012c8cd0
00007000012c8a60:  00000000043c5430  1fe08637dc19adc5
00007000012c8a70:  00000000040b53eb <github.com/bytecodealliance/wasmtime-go.(*Func).Type+43>  0000037f00001fa0
00007000012c8a80:  0000000000000000  0000000000000000
00007000012c8a90:  0000000004019f5e <runtime.(*mcentral).grow+318>  0000000000000000
00007000012c8aa0:  000000c00061a010  000000c000610018
00007000012c8ab0:  0000000000000000  00007000012c8b18
00007000012c8ac0:  0000000047c4c130  00007000012c8cd0
00007000012c8ad0:  4d08e922884200c0  00007000012c8cd0
00007000012c8ae0:  0000000000000008  00007000012c8da8
00007000012c8af0:  00007000012c8d20  00007000012c8b50
00007000012c8b00:  00007000012c8bc0  00000000043c54ef

goroutine 2281 [syscall]:
runtime.cgocall(0x40bd190, 0xc000616d80, 0xc00061a010)
        /nix/store/9xx9iq19j8wm4vaj8042jkzyx06ypa5z-go-1.15.4/share/go/src/runtime/cgocall.go:133 +0x5b fp=0xc000616d50 sp=0xc000616d18 pc=0x4005efb
github.com/bytecodealliance/wasmtime-go._Cfunc_go_wasmtime_func_call(0x47c4d750, 0xc00061a000, 0xc00061a010, 0xc000610018, 0x0)
        _cgo_gotypes.go:718 +0x4e fp=0xc000616d80 sp=0xc000616d50 pc=0x40b12ee
github.com/bytecodealliance/wasmtime-go.(*Func).Call.func3.1(0xc00060e000, 0xc00061a000, 0xc00061a010, 0xc000610018, 0x4d3bb01)
        /Users/stephan/Sources/open-policy-agent/opa/vendor/github.com/bytecodealliance/wasmtime-go/func.go:505 +0x117 fp=0xc000616dc0 sp=0xc000616d80 pc=0x40b9d17
github.com/bytecodealliance/wasmtime-go.(*Func).Call.func3(0xc000610018)
        /Users/stephan/Sources/open-policy-agent/opa/vendor/github.com/bytecodealliance/wasmtime-go/func.go:505 +0x4f fp=0xc000616e00 sp=0xc000616dc0 pc=0x40b9d8f
github.com/bytecodealliance/wasmtime-go.enterWasm(0xc000443b00, 0xc000616f30, 0x0)
        /Users/stephan/Sources/open-policy-agent/opa/vendor/github.com/bytecodealliance/wasmtime-go/func.go:589 +0xf0 fp=0xc000616e60 sp=0xc000616e00 pc=0x40b5e50
github.com/bytecodealliance/wasmtime-go.(*Func).Call(0xc00060e000, 0x0, 0x0, 0x0, 0xc00044c088, 0x4d259d0, 0x4d40960, 0xc0004264d8)
        /Users/stephan/Sources/open-policy-agent/opa/vendor/github.com/bytecodealliance/wasmtime-go/func.go:500 +0x2be fp=0xc000616f68 sp=0xc000616e60 pc=0x40b575e
main.main.func1(0xc0004375c0, 0xc000446870, 0xc00044a3c0)
        /Users/stephan/Sources/open-policy-agent/opa/scratch/issue-3225/main.go:35 +0x93 fp=0xc000616fc8 sp=0xc000616f68 pc=0x40bc513
runtime.goexit()
        /nix/store/9xx9iq19j8wm4vaj8042jkzyx06ypa5z-go-1.15.4/share/go/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc000616fd0 sp=0xc000616fc8 pc=0x40674c1
created by main.main
        /Users/stephan/Sources/open-policy-agent/opa/scratch/issue-3225/main.go:34 +0x276

goroutine 1 [chan receive]:
main.main()
        /Users/stephan/Sources/open-policy-agent/opa/scratch/issue-3225/main.go:45 +0x318

rax    0xffffffffffff7fff
rbx    0x0
rcx    0x8
rdx    0x4a100000
rdi    0xffffffffffff7fff
rsi    0x47c4ba50
rbp    0x7000012c8a10
rsp    0x7000012c8a10
r8     0x4a10001c
r9     0xc000610018
r10    0xc00061a010
r11    0x0
r12    0x7000012c8b50
r13    0x7000012c8a30
r14    0x7000012c8cd0
r15    0x43c5430
rip    0x4a10001a
rflags 0x10246
cs     0x2b
fs     0x0
gs     0x0
exit status 2

[alexcrichton]

@srenatus the bug you're running into is you're concurrently using a Store which is not safe. If you switch the sleep to happening on the goroutine then your example runs safely. The objects connected to Store are not safe to use concurrently on multiple goroutines.

This is what works for me -- https://gist.github.com/alexcrichton/1be9a044be50e95e0584f70170a6d55d

[srenatus]

The objects connected to Store are not safe to use concurrently on multiple goroutines.

💡 Thanks, that's helpful. 😃

[srenatus]

@alexcrichton So just for clarification, the interrupt handle can be used from another goroutine, correct?

Also, I see the same thing happening with your code:

2021/04/20 20:33:42 358: trap: wasm trap: interrupt
wasm backtrace:
    0:   0x21 - <unknown>!<wasm function 0>
SIGILL: illegal instruction
PC=0x32c7001a m=4 sigcode=1
instruction bytes: 0xf 0xb 0x55 0x48 0x89 0xe5 0xff 0xd2 0x48 0x89 0xec 0x5d 0xc3 0x0 0x0 0x0

goroutine 0 [idle]:
runtime: unknown pc 0x32c7001a
stack: frame={sp:0x70000f597a10, fp:0x0} stack=[0x70000f518350,0x70000f597f50)
000070000f597910:  0000000000000000  0000000000000000
000070000f597920:  0000000000000000  0000000000000000
000070000f597930:  0000000000000000  0000000000000000
000070000f597940:  0000000000000000  0000000000000000
000070000f597950:  0000000000000000  0000000000000000
000070000f597960:  0000000000000000  0000000000000000
000070000f597970:  0000000000000000  0000000000000000
000070000f597980:  0000000000000000  0000000000000000
000070000f597990:  0000000000000000  0000000000000000
000070000f5979a0:  0000000000000000  0000000000000000
000070000f5979b0:  0000000000000000  0000000000000000
000070000f5979c0:  0000000000000000  0000000000000000
000070000f5979d0:  0000000000000000  0000000000000000
000070000f5979e0:  0000000000000000  0000000000000000
000070000f5979f0:  0000000000000000  0000000000000000
000070000f597a00:  0000000000000000  0000000000000000
000070000f597a10: <000070000f597a20  0000000032c70022
000070000f597a20:  000070000f597b00  00000000047bb82c
000070000f597a30:  0000000000000000  862faa8e1bcb7c2f
000070000f597a40:  862faa8e1bcb7d1f  000070000f597b50
000070000f597a50:  000070000f597a30  000070000f597cd0
000070000f597a60:  00000000043c5530  862fda8e10e9bf31
000070000f597a70:  0000000000000000  0000037f00001f80
000070000f597a80:  0000000000000000  00007fff201c9020
000070000f597a90:  0000000000000000  0000000000000000
000070000f597aa0:  000000c00022a190  000000c000221a48
000070000f597ab0:  0000000000000000  000070000f597b18
000070000f597ac0:  00000000070d3150  000070000f597cd0
000070000f597ad0:  a4b27d404b7300e3  000070000f597cd0
000070000f597ae0:  0000000000000008  000070000f597da8
000070000f597af0:  000070000f597d20  000070000f597b50
000070000f597b00:  000070000f597bc0  00000000043c55ef
runtime: unknown pc 0x32c7001a
stack: frame={sp:0x70000f597a10, fp:0x0} stack=[0x70000f518350,0x70000f597f50)
000070000f597910:  0000000000000000  0000000000000000
000070000f597920:  0000000000000000  0000000000000000
000070000f597930:  0000000000000000  0000000000000000
000070000f597940:  0000000000000000  0000000000000000
000070000f597950:  0000000000000000  0000000000000000
000070000f597960:  0000000000000000  0000000000000000
000070000f597970:  0000000000000000  0000000000000000
000070000f597980:  0000000000000000  0000000000000000
000070000f597990:  0000000000000000  0000000000000000
000070000f5979a0:  0000000000000000  0000000000000000
000070000f5979b0:  0000000000000000  0000000000000000
000070000f5979c0:  0000000000000000  0000000000000000
000070000f5979d0:  0000000000000000  0000000000000000
000070000f5979e0:  0000000000000000  0000000000000000
000070000f5979f0:  0000000000000000  0000000000000000
000070000f597a00:  0000000000000000  0000000000000000
000070000f597a10: <000070000f597a20  0000000032c70022
000070000f597a20:  000070000f597b00  00000000047bb82c
000070000f597a30:  0000000000000000  862faa8e1bcb7c2f
000070000f597a40:  862faa8e1bcb7d1f  000070000f597b50
000070000f597a50:  000070000f597a30  000070000f597cd0
000070000f597a60:  00000000043c5530  862fda8e10e9bf31
000070000f597a70:  0000000000000000  0000037f00001f80
000070000f597a80:  0000000000000000  00007fff201c9020
000070000f597a90:  0000000000000000  0000000000000000
000070000f597aa0:  000000c00022a190  000000c000221a48
000070000f597ab0:  0000000000000000  000070000f597b18
000070000f597ac0:  00000000070d3150  000070000f597cd0
000070000f597ad0:  a4b27d404b7300e3  000070000f597cd0
000070000f597ae0:  0000000000000008  000070000f597da8
000070000f597af0:  000070000f597d20  000070000f597b50
000070000f597b00:  000070000f597bc0  00000000043c55ef

goroutine 1 [syscall]:
runtime.cgocall(0x40bd290, 0xc0000abce8, 0xc00022a190)
        /nix/store/9xx9iq19j8wm4vaj8042jkzyx06ypa5z-go-1.15.4/share/go/src/runtime/cgocall.go:133 +0x5b fp=0xc0000abcb8 sp=0xc0000abc80 pc=0x400621b
github.com/bytecodealliance/wasmtime-go._Cfunc_go_wasmtime_func_call(0x70d3550, 0xc00022a180, 0xc00022a190, 0xc000221a48, 0x0)
        _cgo_gotypes.go:718 +0x4e fp=0xc0000abce8 sp=0xc0000abcb8 pc=0x40b160e
github.com/bytecodealliance/wasmtime-go.(*Func).Call.func3.1(0xc000226c00, 0xc00022a180, 0xc00022a190, 0xc000221a48, 0x40532db742520c01)
        /Users/stephan/Sources/open-policy-agent/opa/vendor/github.com/bytecodealliance/wasmtime-go/func.go:505 +0x117 fp=0xc0000abd28 sp=0xc0000abce8 pc=0x40ba037
github.com/bytecodealliance/wasmtime-go.(*Func).Call.func3(0xc000221a48)
        /Users/stephan/Sources/open-policy-agent/opa/vendor/github.com/bytecodealliance/wasmtime-go/func.go:505 +0x4f fp=0xc0000abd68 sp=0xc0000abd28 pc=0x40ba0af
github.com/bytecodealliance/wasmtime-go.enterWasm(0xc0001326c0, 0xc0000abe98, 0x0)
        /Users/stephan/Sources/open-policy-agent/opa/vendor/github.com/bytecodealliance/wasmtime-go/func.go:589 +0xf0 fp=0xc0000abdc8 sp=0xc0000abd68 pc=0x40b6170
github.com/bytecodealliance/wasmtime-go.(*Func).Call(0xc000226c00, 0x0, 0x0, 0x0, 0xc0002238e8, 0xc0001303f0, 0x0, 0x0)
        /Users/stephan/Sources/open-policy-agent/opa/vendor/github.com/bytecodealliance/wasmtime-go/func.go:500 +0x2be fp=0xc0000abed0 sp=0xc0000abdc8 pc=0x40b5a7e
main.main()
        /Users/stephan/Sources/open-policy-agent/opa/scratch/issue-3225/main2.go:39 +0x215 fp=0xc0000abf88 sp=0xc0000abed0 pc=0x40bc495
runtime.main()
        /nix/store/9xx9iq19j8wm4vaj8042jkzyx06ypa5z-go-1.15.4/share/go/src/runtime/proc.go:204 +0x209 fp=0xc0000abfe0 sp=0xc0000abf88 pc=0x4038949
runtime.goexit()
        /nix/store/9xx9iq19j8wm4vaj8042jkzyx06ypa5z-go-1.15.4/share/go/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc0000abfe8 sp=0xc0000abfe0 pc=0x40677e1

rax    0xffffffffffff7fff
rbx    0x0
rcx    0x8
rdx    0x32c70000
rdi    0xffffffffffff7fff
rsi    0x70d2a70
rbp    0x70000f597a10
rsp    0x70000f597a10
r8     0x32c7001c
r9     0xc000221a48
r10    0xc00022a190
r11    0x0
r12    0x70000f597b50
r13    0x70000f597a30
r14    0x70000f597cd0
r15    0x43c5530
rip    0x32c7001a
rflags 0x10246
cs     0x2b
fs     0x0
gs     0x0
exit status 2

[alexcrichton]

Hm yes I think I've seen that too from time to time and is likely related to the Go runtime, I'll need to dig in further to see what's going on though.