mirror of https://go.googlesource.com/go
422 lines
17 KiB
Go
422 lines
17 KiB
Go
// Copyright 2014 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package runtime
|
|
|
|
import (
|
|
"internal/abi"
|
|
"unsafe"
|
|
)
|
|
|
|
// Should be a built-in for unsafe.Pointer?
|
|
//
|
|
//go:nosplit
|
|
func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
|
|
return unsafe.Pointer(uintptr(p) + x)
|
|
}
|
|
|
|
// getg returns the pointer to the current g.
|
|
// The compiler rewrites calls to this function into instructions
|
|
// that fetch the g directly (from TLS or from the dedicated register).
|
|
func getg() *g
|
|
|
|
// mcall switches from the g to the g0 stack and invokes fn(g),
|
|
// where g is the goroutine that made the call.
|
|
// mcall saves g's current PC/SP in g->sched so that it can be restored later.
|
|
// It is up to fn to arrange for that later execution, typically by recording
|
|
// g in a data structure, causing something to call ready(g) later.
|
|
// mcall returns to the original goroutine g later, when g has been rescheduled.
|
|
// fn must not return at all; typically it ends by calling schedule, to let the m
|
|
// run other goroutines.
|
|
//
|
|
// mcall can only be called from g stacks (not g0, not gsignal).
|
|
//
|
|
// This must NOT be go:noescape: if fn is a stack-allocated closure,
|
|
// fn puts g on a run queue, and g executes before fn returns, the
|
|
// closure will be invalidated while it is still executing.
|
|
func mcall(fn func(*g))
|
|
|
|
// systemstack runs fn on a system stack.
|
|
// If systemstack is called from the per-OS-thread (g0) stack, or
|
|
// if systemstack is called from the signal handling (gsignal) stack,
|
|
// systemstack calls fn directly and returns.
|
|
// Otherwise, systemstack is being called from the limited stack
|
|
// of an ordinary goroutine. In this case, systemstack switches
|
|
// to the per-OS-thread stack, calls fn, and switches back.
|
|
// It is common to use a func literal as the argument, in order
|
|
// to share inputs and outputs with the code around the call
|
|
// to system stack:
|
|
//
|
|
// ... set up y ...
|
|
// systemstack(func() {
|
|
// x = bigcall(y)
|
|
// })
|
|
// ... use x ...
|
|
//
|
|
//go:noescape
|
|
func systemstack(fn func())
|
|
|
|
//go:nosplit
|
|
//go:nowritebarrierrec
|
|
func badsystemstack() {
|
|
writeErrStr("fatal: systemstack called from unexpected goroutine")
|
|
}
|
|
|
|
// memclrNoHeapPointers clears n bytes starting at ptr.
|
|
//
|
|
// Usually you should use typedmemclr. memclrNoHeapPointers should be
|
|
// used only when the caller knows that *ptr contains no heap pointers
|
|
// because either:
|
|
//
|
|
// *ptr is initialized memory and its type is pointer-free, or
|
|
//
|
|
// *ptr is uninitialized memory (e.g., memory that's being reused
|
|
// for a new allocation) and hence contains only "junk".
|
|
//
|
|
// memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n
|
|
// is a multiple of the pointer size, then any pointer-aligned,
|
|
// pointer-sized portion is cleared atomically. Despite the function
|
|
// name, this is necessary because this function is the underlying
|
|
// implementation of typedmemclr and memclrHasPointers. See the doc of
|
|
// memmove for more details.
|
|
//
|
|
// The (CPU-specific) implementations of this function are in memclr_*.s.
|
|
//
|
|
//go:noescape
|
|
func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
|
|
|
|
//go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers
|
|
func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) {
|
|
memclrNoHeapPointers(ptr, n)
|
|
}
|
|
|
|
// memmove copies n bytes from "from" to "to".
|
|
//
|
|
// memmove ensures that any pointer in "from" is written to "to" with
|
|
// an indivisible write, so that racy reads cannot observe a
|
|
// half-written pointer. This is necessary to prevent the garbage
|
|
// collector from observing invalid pointers, and differs from memmove
|
|
// in unmanaged languages. However, memmove is only required to do
|
|
// this if "from" and "to" may contain pointers, which can only be the
|
|
// case if "from", "to", and "n" are all be word-aligned.
|
|
//
|
|
// Implementations are in memmove_*.s.
|
|
//
|
|
//go:noescape
|
|
func memmove(to, from unsafe.Pointer, n uintptr)
|
|
|
|
// Outside assembly calls memmove. Make sure it has ABI wrappers.
|
|
//
|
|
//go:linkname memmove
|
|
|
|
//go:linkname reflect_memmove reflect.memmove
|
|
func reflect_memmove(to, from unsafe.Pointer, n uintptr) {
|
|
memmove(to, from, n)
|
|
}
|
|
|
|
// exported value for testing
|
|
const hashLoad = float32(loadFactorNum) / float32(loadFactorDen)
|
|
|
|
// in internal/bytealg/equal_*.s
|
|
//
|
|
//go:noescape
|
|
func memequal(a, b unsafe.Pointer, size uintptr) bool
|
|
|
|
// noescape hides a pointer from escape analysis. noescape is
|
|
// the identity function but escape analysis doesn't think the
|
|
// output depends on the input. noescape is inlined and currently
|
|
// compiles down to zero instructions.
|
|
// USE CAREFULLY!
|
|
//
|
|
//go:nosplit
|
|
func noescape(p unsafe.Pointer) unsafe.Pointer {
|
|
x := uintptr(p)
|
|
return unsafe.Pointer(x ^ 0)
|
|
}
|
|
|
|
// noEscapePtr hides a pointer from escape analysis. See noescape.
|
|
// USE CAREFULLY!
|
|
//
|
|
//go:nosplit
|
|
func noEscapePtr[T any](p *T) *T {
|
|
x := uintptr(unsafe.Pointer(p))
|
|
return (*T)(unsafe.Pointer(x ^ 0))
|
|
}
|
|
|
|
// Not all cgocallback frames are actually cgocallback,
|
|
// so not all have these arguments. Mark them uintptr so that the GC
|
|
// does not misinterpret memory when the arguments are not present.
|
|
// cgocallback is not called from Go, only from crosscall2.
|
|
// This in turn calls cgocallbackg, which is where we'll find
|
|
// pointer-declared arguments.
|
|
//
|
|
// When fn is nil (frame is saved g), call dropm instead,
|
|
// this is used when the C thread is exiting.
|
|
func cgocallback(fn, frame, ctxt uintptr)
|
|
|
|
func gogo(buf *gobuf)
|
|
|
|
func asminit()
|
|
func setg(gg *g)
|
|
func breakpoint()
|
|
|
|
// reflectcall calls fn with arguments described by stackArgs, stackArgsSize,
|
|
// frameSize, and regArgs.
|
|
//
|
|
// Arguments passed on the stack and space for return values passed on the stack
|
|
// must be laid out at the space pointed to by stackArgs (with total length
|
|
// stackArgsSize) according to the ABI.
|
|
//
|
|
// stackRetOffset must be some value <= stackArgsSize that indicates the
|
|
// offset within stackArgs where the return value space begins.
|
|
//
|
|
// frameSize is the total size of the argument frame at stackArgs and must
|
|
// therefore be >= stackArgsSize. It must include additional space for spilling
|
|
// register arguments for stack growth and preemption.
|
|
//
|
|
// TODO(mknyszek): Once we don't need the additional spill space, remove frameSize,
|
|
// since frameSize will be redundant with stackArgsSize.
|
|
//
|
|
// Arguments passed in registers must be laid out in regArgs according to the ABI.
|
|
// regArgs will hold any return values passed in registers after the call.
|
|
//
|
|
// reflectcall copies stack arguments from stackArgs to the goroutine stack, and
|
|
// then copies back stackArgsSize-stackRetOffset bytes back to the return space
|
|
// in stackArgs once fn has completed. It also "unspills" argument registers from
|
|
// regArgs before calling fn, and spills them back into regArgs immediately
|
|
// following the call to fn. If there are results being returned on the stack,
|
|
// the caller should pass the argument frame type as stackArgsType so that
|
|
// reflectcall can execute appropriate write barriers during the copy.
|
|
//
|
|
// reflectcall expects regArgs.ReturnIsPtr to be populated indicating which
|
|
// registers on the return path will contain Go pointers. It will then store
|
|
// these pointers in regArgs.Ptrs such that they are visible to the GC.
|
|
//
|
|
// Package reflect passes a frame type. In package runtime, there is only
|
|
// one call that copies results back, in callbackWrap in syscall_windows.go, and it
|
|
// does NOT pass a frame type, meaning there are no write barriers invoked. See that
|
|
// call site for justification.
|
|
//
|
|
// Package reflect accesses this symbol through a linkname.
|
|
//
|
|
// Arguments passed through to reflectcall do not escape. The type is used
|
|
// only in a very limited callee of reflectcall, the stackArgs are copied, and
|
|
// regArgs is only used in the reflectcall frame.
|
|
//
|
|
//go:noescape
|
|
func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
|
|
func procyield(cycles uint32)
|
|
|
|
type neverCallThisFunction struct{}
|
|
|
|
// goexit is the return stub at the top of every goroutine call stack.
|
|
// Each goroutine stack is constructed as if goexit called the
|
|
// goroutine's entry point function, so that when the entry point
|
|
// function returns, it will return to goexit, which will call goexit1
|
|
// to perform the actual exit.
|
|
//
|
|
// This function must never be called directly. Call goexit1 instead.
|
|
// gentraceback assumes that goexit terminates the stack. A direct
|
|
// call on the stack will cause gentraceback to stop walking the stack
|
|
// prematurely and if there is leftover state it may panic.
|
|
func goexit(neverCallThisFunction)
|
|
|
|
// publicationBarrier performs a store/store barrier (a "publication"
|
|
// or "export" barrier). Some form of synchronization is required
|
|
// between initializing an object and making that object accessible to
|
|
// another processor. Without synchronization, the initialization
|
|
// writes and the "publication" write may be reordered, allowing the
|
|
// other processor to follow the pointer and observe an uninitialized
|
|
// object. In general, higher-level synchronization should be used,
|
|
// such as locking or an atomic pointer write. publicationBarrier is
|
|
// for when those aren't an option, such as in the implementation of
|
|
// the memory manager.
|
|
//
|
|
// There's no corresponding barrier for the read side because the read
|
|
// side naturally has a data dependency order. All architectures that
|
|
// Go supports or seems likely to ever support automatically enforce
|
|
// data dependency ordering.
|
|
func publicationBarrier()
|
|
|
|
// getcallerpc returns the program counter (PC) of its caller's caller.
|
|
// getcallersp returns the stack pointer (SP) of its caller's caller.
|
|
// The implementation may be a compiler intrinsic; there is not
|
|
// necessarily code implementing this on every platform.
|
|
//
|
|
// For example:
|
|
//
|
|
// func f(arg1, arg2, arg3 int) {
|
|
// pc := getcallerpc()
|
|
// sp := getcallersp()
|
|
// }
|
|
//
|
|
// These two lines find the PC and SP immediately following
|
|
// the call to f (where f will return).
|
|
//
|
|
// The call to getcallerpc and getcallersp must be done in the
|
|
// frame being asked about.
|
|
//
|
|
// The result of getcallersp is correct at the time of the return,
|
|
// but it may be invalidated by any subsequent call to a function
|
|
// that might relocate the stack in order to grow or shrink it.
|
|
// A general rule is that the result of getcallersp should be used
|
|
// immediately and can only be passed to nosplit functions.
|
|
|
|
//go:noescape
|
|
func getcallerpc() uintptr
|
|
|
|
//go:noescape
|
|
func getcallersp() uintptr // implemented as an intrinsic on all platforms
|
|
|
|
// getclosureptr returns the pointer to the current closure.
|
|
// getclosureptr can only be used in an assignment statement
|
|
// at the entry of a function. Moreover, go:nosplit directive
|
|
// must be specified at the declaration of caller function,
|
|
// so that the function prolog does not clobber the closure register.
|
|
// for example:
|
|
//
|
|
// //go:nosplit
|
|
// func f(arg1, arg2, arg3 int) {
|
|
// dx := getclosureptr()
|
|
// }
|
|
//
|
|
// The compiler rewrites calls to this function into instructions that fetch the
|
|
// pointer from a well-known register (DX on x86 architecture, etc.) directly.
|
|
//
|
|
// WARNING: PGO-based devirtualization cannot detect that caller of
|
|
// getclosureptr require closure context, and thus must maintain a list of
|
|
// these functions, which is in
|
|
// cmd/compile/internal/devirtualize/pgo.maybeDevirtualizeFunctionCall.
|
|
func getclosureptr() uintptr
|
|
|
|
//go:noescape
|
|
func asmcgocall(fn, arg unsafe.Pointer) int32
|
|
|
|
func morestack()
|
|
func morestack_noctxt()
|
|
func rt0_go()
|
|
|
|
// return0 is a stub used to return 0 from deferproc.
|
|
// It is called at the very end of deferproc to signal
|
|
// the calling Go function that it should not jump
|
|
// to deferreturn.
|
|
// in asm_*.s
|
|
func return0()
|
|
|
|
// in asm_*.s
|
|
// not called directly; definitions here supply type information for traceback.
|
|
// These must have the same signature (arg pointer map) as reflectcall.
|
|
func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
|
|
|
|
func systemstack_switch()
|
|
|
|
// alignUp rounds n up to a multiple of a. a must be a power of 2.
|
|
//
|
|
//go:nosplit
|
|
func alignUp(n, a uintptr) uintptr {
|
|
return (n + a - 1) &^ (a - 1)
|
|
}
|
|
|
|
// alignDown rounds n down to a multiple of a. a must be a power of 2.
|
|
//
|
|
//go:nosplit
|
|
func alignDown(n, a uintptr) uintptr {
|
|
return n &^ (a - 1)
|
|
}
|
|
|
|
// divRoundUp returns ceil(n / a).
|
|
func divRoundUp(n, a uintptr) uintptr {
|
|
// a is generally a power of two. This will get inlined and
|
|
// the compiler will optimize the division.
|
|
return (n + a - 1) / a
|
|
}
|
|
|
|
// checkASM reports whether assembly runtime checks have passed.
|
|
func checkASM() bool
|
|
|
|
func memequal_varlen(a, b unsafe.Pointer) bool
|
|
|
|
// bool2int returns 0 if x is false or 1 if x is true.
|
|
func bool2int(x bool) int {
|
|
// Avoid branches. In the SSA compiler, this compiles to
|
|
// exactly what you would want it to.
|
|
return int(*(*uint8)(unsafe.Pointer(&x)))
|
|
}
|
|
|
|
// abort crashes the runtime in situations where even throw might not
|
|
// work. In general it should do something a debugger will recognize
|
|
// (e.g., an INT3 on x86). A crash in abort is recognized by the
|
|
// signal handler, which will attempt to tear down the runtime
|
|
// immediately.
|
|
func abort()
|
|
|
|
// Called from compiled code; declared for vet; do NOT call from Go.
|
|
func gcWriteBarrier1()
|
|
func gcWriteBarrier2()
|
|
func gcWriteBarrier3()
|
|
func gcWriteBarrier4()
|
|
func gcWriteBarrier5()
|
|
func gcWriteBarrier6()
|
|
func gcWriteBarrier7()
|
|
func gcWriteBarrier8()
|
|
func duffzero()
|
|
func duffcopy()
|
|
|
|
// Called from linker-generated .initarray; declared for go vet; do NOT call from Go.
|
|
func addmoduledata()
|
|
|
|
// Injected by the signal handler for panicking signals.
|
|
// Initializes any registers that have fixed meaning at calls but
|
|
// are scratch in bodies and calls sigpanic.
|
|
// On many platforms it just jumps to sigpanic.
|
|
func sigpanic0()
|
|
|
|
// intArgRegs is used by the various register assignment
|
|
// algorithm implementations in the runtime. These include:.
|
|
// - Finalizers (mfinal.go)
|
|
// - Windows callbacks (syscall_windows.go)
|
|
//
|
|
// Both are stripped-down versions of the algorithm since they
|
|
// only have to deal with a subset of cases (finalizers only
|
|
// take a pointer or interface argument, Go Windows callbacks
|
|
// don't support floating point).
|
|
//
|
|
// It should be modified with care and are generally only
|
|
// modified when testing this package.
|
|
//
|
|
// It should never be set higher than its internal/abi
|
|
// constant counterparts, because the system relies on a
|
|
// structure that is at least large enough to hold the
|
|
// registers the system supports.
|
|
//
|
|
// Protected by finlock.
|
|
var intArgRegs = abi.IntArgRegs
|