mirror of https://go.googlesource.com/go
174 lines
4.6 KiB
Go
174 lines
4.6 KiB
Go
// Copyright 2013 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
//go:build linux
|
|
|
|
package runtime
|
|
|
|
import (
|
|
"internal/runtime/atomic"
|
|
"internal/runtime/syscall"
|
|
"unsafe"
|
|
)
|
|
|
|
var (
|
|
epfd int32 = -1 // epoll descriptor
|
|
netpollEventFd uintptr // eventfd for netpollBreak
|
|
netpollWakeSig atomic.Uint32 // used to avoid duplicate calls of netpollBreak
|
|
)
|
|
|
|
func netpollinit() {
|
|
var errno uintptr
|
|
epfd, errno = syscall.EpollCreate1(syscall.EPOLL_CLOEXEC)
|
|
if errno != 0 {
|
|
println("runtime: epollcreate failed with", errno)
|
|
throw("runtime: netpollinit failed")
|
|
}
|
|
efd, errno := syscall.Eventfd(0, syscall.EFD_CLOEXEC|syscall.EFD_NONBLOCK)
|
|
if errno != 0 {
|
|
println("runtime: eventfd failed with", -errno)
|
|
throw("runtime: eventfd failed")
|
|
}
|
|
ev := syscall.EpollEvent{
|
|
Events: syscall.EPOLLIN,
|
|
}
|
|
*(**uintptr)(unsafe.Pointer(&ev.Data)) = &netpollEventFd
|
|
errno = syscall.EpollCtl(epfd, syscall.EPOLL_CTL_ADD, efd, &ev)
|
|
if errno != 0 {
|
|
println("runtime: epollctl failed with", errno)
|
|
throw("runtime: epollctl failed")
|
|
}
|
|
netpollEventFd = uintptr(efd)
|
|
}
|
|
|
|
func netpollIsPollDescriptor(fd uintptr) bool {
|
|
return fd == uintptr(epfd) || fd == netpollEventFd
|
|
}
|
|
|
|
func netpollopen(fd uintptr, pd *pollDesc) uintptr {
|
|
var ev syscall.EpollEvent
|
|
ev.Events = syscall.EPOLLIN | syscall.EPOLLOUT | syscall.EPOLLRDHUP | syscall.EPOLLET
|
|
tp := taggedPointerPack(unsafe.Pointer(pd), pd.fdseq.Load())
|
|
*(*taggedPointer)(unsafe.Pointer(&ev.Data)) = tp
|
|
return syscall.EpollCtl(epfd, syscall.EPOLL_CTL_ADD, int32(fd), &ev)
|
|
}
|
|
|
|
func netpollclose(fd uintptr) uintptr {
|
|
var ev syscall.EpollEvent
|
|
return syscall.EpollCtl(epfd, syscall.EPOLL_CTL_DEL, int32(fd), &ev)
|
|
}
|
|
|
|
func netpollarm(pd *pollDesc, mode int) {
|
|
throw("runtime: unused")
|
|
}
|
|
|
|
// netpollBreak interrupts an epollwait.
|
|
func netpollBreak() {
|
|
// Failing to cas indicates there is an in-flight wakeup, so we're done here.
|
|
if !netpollWakeSig.CompareAndSwap(0, 1) {
|
|
return
|
|
}
|
|
|
|
var one uint64 = 1
|
|
oneSize := int32(unsafe.Sizeof(one))
|
|
for {
|
|
n := write(netpollEventFd, noescape(unsafe.Pointer(&one)), oneSize)
|
|
if n == oneSize {
|
|
break
|
|
}
|
|
if n == -_EINTR {
|
|
continue
|
|
}
|
|
if n == -_EAGAIN {
|
|
return
|
|
}
|
|
println("runtime: netpollBreak write failed with", -n)
|
|
throw("runtime: netpollBreak write failed")
|
|
}
|
|
}
|
|
|
|
// netpoll checks for ready network connections.
|
|
// Returns list of goroutines that become runnable.
|
|
// delay < 0: blocks indefinitely
|
|
// delay == 0: does not block, just polls
|
|
// delay > 0: block for up to that many nanoseconds
|
|
func netpoll(delay int64) (gList, int32) {
|
|
if epfd == -1 {
|
|
return gList{}, 0
|
|
}
|
|
var waitms int32
|
|
if delay < 0 {
|
|
waitms = -1
|
|
} else if delay == 0 {
|
|
waitms = 0
|
|
} else if delay < 1e6 {
|
|
waitms = 1
|
|
} else if delay < 1e15 {
|
|
waitms = int32(delay / 1e6)
|
|
} else {
|
|
// An arbitrary cap on how long to wait for a timer.
|
|
// 1e9 ms == ~11.5 days.
|
|
waitms = 1e9
|
|
}
|
|
var events [128]syscall.EpollEvent
|
|
retry:
|
|
n, errno := syscall.EpollWait(epfd, events[:], int32(len(events)), waitms)
|
|
if errno != 0 {
|
|
if errno != _EINTR {
|
|
println("runtime: epollwait on fd", epfd, "failed with", errno)
|
|
throw("runtime: netpoll failed")
|
|
}
|
|
// If a timed sleep was interrupted, just return to
|
|
// recalculate how long we should sleep now.
|
|
if waitms > 0 {
|
|
return gList{}, 0
|
|
}
|
|
goto retry
|
|
}
|
|
var toRun gList
|
|
delta := int32(0)
|
|
for i := int32(0); i < n; i++ {
|
|
ev := events[i]
|
|
if ev.Events == 0 {
|
|
continue
|
|
}
|
|
|
|
if *(**uintptr)(unsafe.Pointer(&ev.Data)) == &netpollEventFd {
|
|
if ev.Events != syscall.EPOLLIN {
|
|
println("runtime: netpoll: eventfd ready for", ev.Events)
|
|
throw("runtime: netpoll: eventfd ready for something unexpected")
|
|
}
|
|
if delay != 0 {
|
|
// netpollBreak could be picked up by a
|
|
// nonblocking poll. Only read the 8-byte
|
|
// integer if blocking.
|
|
// Since EFD_SEMAPHORE was not specified,
|
|
// the eventfd counter will be reset to 0.
|
|
var one uint64
|
|
read(int32(netpollEventFd), noescape(unsafe.Pointer(&one)), int32(unsafe.Sizeof(one)))
|
|
netpollWakeSig.Store(0)
|
|
}
|
|
continue
|
|
}
|
|
|
|
var mode int32
|
|
if ev.Events&(syscall.EPOLLIN|syscall.EPOLLRDHUP|syscall.EPOLLHUP|syscall.EPOLLERR) != 0 {
|
|
mode += 'r'
|
|
}
|
|
if ev.Events&(syscall.EPOLLOUT|syscall.EPOLLHUP|syscall.EPOLLERR) != 0 {
|
|
mode += 'w'
|
|
}
|
|
if mode != 0 {
|
|
tp := *(*taggedPointer)(unsafe.Pointer(&ev.Data))
|
|
pd := (*pollDesc)(tp.pointer())
|
|
tag := tp.tag()
|
|
if pd.fdseq.Load() == tag {
|
|
pd.setEventErr(ev.Events == syscall.EPOLLERR, tag)
|
|
delta += netpollready(&toRun, pd, mode)
|
|
}
|
|
}
|
|
}
|
|
return toRun, delta
|
|
}
|