Source file
src/runtime/runtime2.go
1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 package runtime
6
7 import (
8 "internal/abi"
9 "internal/chacha8rand"
10 "internal/goarch"
11 "internal/runtime/atomic"
12 "internal/runtime/sys"
13 "unsafe"
14 )
15
16 // defined constants
17 const (
18 // G status
19 //
20 // Beyond indicating the general state of a G, the G status
21 // acts like a lock on the goroutine's stack (and hence its
22 // ability to execute user code).
23 //
24 // If you add to this list, add to the list
25 // of "okay during garbage collection" status
26 // in mgcmark.go too.
27 //
28 // TODO(austin): The _Gscan bit could be much lighter-weight.
29 // For example, we could choose not to run _Gscanrunnable
30 // goroutines found in the run queue, rather than CAS-looping
31 // until they become _Grunnable. And transitions like
32 // _Gscanwaiting -> _Gscanrunnable are actually okay because
33 // they don't affect stack ownership.
34
35 // _Gidle means this goroutine was just allocated and has not
36 // yet been initialized.
37 _Gidle = iota // 0
38
39 // _Grunnable means this goroutine is on a run queue. It is
40 // not currently executing user code. The stack is not owned.
41 _Grunnable // 1
42
43 // _Grunning means this goroutine may execute user code. The
44 // stack is owned by this goroutine. It is not on a run queue.
45 // It is assigned an M (g.m is valid) and it usually has a P
46 // (g.m.p is valid), but there are small windows of time where
47 // it might not, namely upon entering and exiting _Gsyscall.
48 _Grunning // 2
49
50 // _Gsyscall means this goroutine is executing a system call.
51 // It is not executing user code. The stack is owned by this
52 // goroutine. It is not on a run queue. It is assigned an M.
53 // It may have a P attached, but it does not own it. Code
54 // executing in this state must not touch g.m.p.
55 _Gsyscall // 3
56
57 // _Gwaiting means this goroutine is blocked in the runtime.
58 // It is not executing user code. It is not on a run queue,
59 // but should be recorded somewhere (e.g., a channel wait
60 // queue) so it can be ready()d when necessary. The stack is
61 // not owned *except* that a channel operation may read or
62 // write parts of the stack under the appropriate channel
63 // lock. Otherwise, it is not safe to access the stack after a
64 // goroutine enters _Gwaiting (e.g., it may get moved).
65 _Gwaiting // 4
66
67 // _Gmoribund_unused is currently unused, but hardcoded in gdb
68 // scripts.
69 _Gmoribund_unused // 5
70
71 // _Gdead means this goroutine is currently unused. It may be
72 // just exited, on a free list, or just being initialized. It
73 // is not executing user code. It may or may not have a stack
74 // allocated. The G and its stack (if any) are owned by the M
75 // that is exiting the G or that obtained the G from the free
76 // list.
77 _Gdead // 6
78
79 // _Genqueue_unused is currently unused.
80 _Genqueue_unused // 7
81
82 // _Gcopystack means this goroutine's stack is being moved. It
83 // is not executing user code and is not on a run queue. The
84 // stack is owned by the goroutine that put it in _Gcopystack.
85 _Gcopystack // 8
86
87 // _Gpreempted means this goroutine stopped itself for a
88 // suspendG preemption. It is like _Gwaiting, but nothing is
89 // yet responsible for ready()ing it. Some suspendG must CAS
90 // the status to _Gwaiting to take responsibility for
91 // ready()ing this G.
92 _Gpreempted // 9
93
94 // _Gleaked represents a leaked goroutine caught by the GC.
95 _Gleaked // 10
96
97 // _Gdeadextra is a _Gdead goroutine that's attached to an extra M
98 // used for cgo callbacks.
99 _Gdeadextra // 11
100
101 // _Gscan combined with one of the above states other than
102 // _Grunning indicates that GC is scanning the stack. The
103 // goroutine is not executing user code and the stack is owned
104 // by the goroutine that set the _Gscan bit.
105 //
106 // _Gscanrunning is different: it is used to briefly block
107 // state transitions while GC signals the G to scan its own
108 // stack. This is otherwise like _Grunning.
109 //
110 // atomicstatus&~Gscan gives the state the goroutine will
111 // return to when the scan completes.
112 _Gscan = 0x1000
113 _Gscanrunnable = _Gscan + _Grunnable // 0x1001
114 _Gscanrunning = _Gscan + _Grunning // 0x1002
115 _Gscansyscall = _Gscan + _Gsyscall // 0x1003
116 _Gscanwaiting = _Gscan + _Gwaiting // 0x1004
117 _Gscanpreempted = _Gscan + _Gpreempted // 0x1009
118 _Gscanleaked = _Gscan + _Gleaked // 0x100a
119 _Gscandeadextra = _Gscan + _Gdeadextra // 0x100b
120 )
121
122 const (
123 // P status
124
125 // _Pidle means a P is not being used to run user code or the
126 // scheduler. Typically, it's on the idle P list and available
127 // to the scheduler, but it may just be transitioning between
128 // other states.
129 //
130 // The P is owned by the idle list or by whatever is
131 // transitioning its state. Its run queue is empty.
132 _Pidle = iota
133
134 // _Prunning means a P is owned by an M and is being used to
135 // run user code or the scheduler. Only the M that owns this P
136 // is allowed to change the P's status from _Prunning. The M
137 // may transition the P to _Pidle (if it has no more work to
138 // do), or _Pgcstop (to halt for the GC). The M may also hand
139 // ownership of the P off directly to another M (for example,
140 // to schedule a locked G).
141 _Prunning
142
143 // _Psyscall_unused is a now-defunct state for a P. A P is
144 // identified as "in a system call" by looking at the goroutine's
145 // state.
146 _Psyscall_unused
147
148 // _Pgcstop means a P is halted for STW and owned by the M
149 // that stopped the world. The M that stopped the world
150 // continues to use its P, even in _Pgcstop. Transitioning
151 // from _Prunning to _Pgcstop causes an M to release its P and
152 // park.
153 //
154 // The P retains its run queue and startTheWorld will restart
155 // the scheduler on Ps with non-empty run queues.
156 _Pgcstop
157
158 // _Pdead means a P is no longer used (GOMAXPROCS shrank). We
159 // reuse Ps if GOMAXPROCS increases. A dead P is mostly
160 // stripped of its resources, though a few things remain
161 // (e.g., trace buffers).
162 _Pdead
163 )
164
165 // Mutual exclusion locks. In the uncontended case,
166 // as fast as spin locks (just a few user-level instructions),
167 // but on the contention path they sleep in the kernel.
168 // A zeroed Mutex is unlocked (no need to initialize each lock).
169 // Initialization is helpful for static lock ranking, but not required.
170 type mutex struct {
171 // Empty struct if lock ranking is disabled, otherwise includes the lock rank
172 lockRankStruct
173 // Futex-based impl treats it as uint32 key,
174 // while sema-based impl as M* waitm.
175 // Used to be a union, but unions break precise GC.
176 key uintptr
177 }
178
179 type funcval struct {
180 fn uintptr
181 // variable-size, fn-specific data here
182 }
183
184 type iface struct {
185 tab *itab
186 data unsafe.Pointer
187 }
188
189 type eface struct {
190 _type *_type
191 data unsafe.Pointer
192 }
193
194 func efaceOf(ep *any) *eface {
195 return (*eface)(unsafe.Pointer(ep))
196 }
197
198 // The guintptr, muintptr, and puintptr are all used to bypass write barriers.
199 // It is particularly important to avoid write barriers when the current P has
200 // been released, because the GC thinks the world is stopped, and an
201 // unexpected write barrier would not be synchronized with the GC,
202 // which can lead to a half-executed write barrier that has marked the object
203 // but not queued it. If the GC skips the object and completes before the
204 // queuing can occur, it will incorrectly free the object.
205 //
206 // We tried using special assignment functions invoked only when not
207 // holding a running P, but then some updates to a particular memory
208 // word went through write barriers and some did not. This breaks the
209 // write barrier shadow checking mode, and it is also scary: better to have
210 // a word that is completely ignored by the GC than to have one for which
211 // only a few updates are ignored.
212 //
213 // Gs and Ps are always reachable via true pointers in the
214 // allgs and allp lists or (during allocation before they reach those lists)
215 // from stack variables.
216 //
217 // Ms are always reachable via true pointers either from allm or
218 // freem. Unlike Gs and Ps we do free Ms, so it's important that
219 // nothing ever hold an muintptr across a safe point.
220
221 // A guintptr holds a goroutine pointer, but typed as a uintptr
222 // to bypass write barriers. It is used in the Gobuf goroutine state
223 // and in scheduling lists that are manipulated without a P.
224 //
225 // The Gobuf.g goroutine pointer is almost always updated by assembly code.
226 // In one of the few places it is updated by Go code - func save - it must be
227 // treated as a uintptr to avoid a write barrier being emitted at a bad time.
228 // Instead of figuring out how to emit the write barriers missing in the
229 // assembly manipulation, we change the type of the field to uintptr,
230 // so that it does not require write barriers at all.
231 //
232 // Goroutine structs are published in the allg list and never freed.
233 // That will keep the goroutine structs from being collected.
234 // There is never a time that Gobuf.g's contain the only references
235 // to a goroutine: the publishing of the goroutine in allg comes first.
236 // Goroutine pointers are also kept in non-GC-visible places like TLS,
237 // so I can't see them ever moving. If we did want to start moving data
238 // in the GC, we'd need to allocate the goroutine structs from an
239 // alternate arena. Using guintptr doesn't make that problem any worse.
240 // Note that pollDesc.rg, pollDesc.wg also store g in uintptr form,
241 // so they would need to be updated too if g's start moving.
242 type guintptr uintptr
243
244 //go:nosplit
245 func (gp guintptr) ptr() *g { return (*g)(unsafe.Pointer(gp)) }
246
247 //go:nosplit
248 func (gp *guintptr) set(g *g) { *gp = guintptr(unsafe.Pointer(g)) }
249
250 //go:nosplit
251 func (gp *guintptr) cas(old, new guintptr) bool {
252 return atomic.Casuintptr((*uintptr)(unsafe.Pointer(gp)), uintptr(old), uintptr(new))
253 }
254
255 //go:nosplit
256 func (gp *g) guintptr() guintptr {
257 return guintptr(unsafe.Pointer(gp))
258 }
259
260 // setGNoWB performs *gp = new without a write barrier.
261 // For times when it's impractical to use a guintptr.
262 //
263 //go:nosplit
264 //go:nowritebarrier
265 func setGNoWB(gp **g, new *g) {
266 (*guintptr)(unsafe.Pointer(gp)).set(new)
267 }
268
269 type puintptr uintptr
270
271 //go:nosplit
272 func (pp puintptr) ptr() *p { return (*p)(unsafe.Pointer(pp)) }
273
274 //go:nosplit
275 func (pp *puintptr) set(p *p) { *pp = puintptr(unsafe.Pointer(p)) }
276
277 // muintptr is a *m that is not tracked by the garbage collector.
278 //
279 // Because we do free Ms, there are some additional constrains on
280 // muintptrs:
281 //
282 // 1. Never hold an muintptr locally across a safe point.
283 //
284 // 2. Any muintptr in the heap must be owned by the M itself so it can
285 // ensure it is not in use when the last true *m is released.
286 type muintptr uintptr
287
288 //go:nosplit
289 func (mp muintptr) ptr() *m { return (*m)(unsafe.Pointer(mp)) }
290
291 //go:nosplit
292 func (mp *muintptr) set(m *m) { *mp = muintptr(unsafe.Pointer(m)) }
293
294 // setMNoWB performs *mp = new without a write barrier.
295 // For times when it's impractical to use an muintptr.
296 //
297 //go:nosplit
298 //go:nowritebarrier
299 func setMNoWB(mp **m, new *m) {
300 (*muintptr)(unsafe.Pointer(mp)).set(new)
301 }
302
303 type gobuf struct {
304 // ctxt is unusual with respect to GC: it may be a
305 // heap-allocated funcval, so GC needs to track it, but it
306 // needs to be set and cleared from assembly, where it's
307 // difficult to have write barriers. However, ctxt is really a
308 // saved, live register, and we only ever exchange it between
309 // the real register and the gobuf. Hence, we treat it as a
310 // root during stack scanning, which means assembly that saves
311 // and restores it doesn't need write barriers. It's still
312 // typed as a pointer so that any other writes from Go get
313 // write barriers.
314 sp uintptr
315 pc uintptr
316 g guintptr
317 ctxt unsafe.Pointer
318 lr uintptr
319 bp uintptr // for framepointer-enabled architectures
320 }
321
322 // maybeTraceablePtr is a special pointer that is conditionally trackable
323 // by the GC. It consists of an address as a uintptr (vu) and a pointer
324 // to a data element (vp).
325 //
326 // maybeTraceablePtr values can be in one of three states:
327 // 1. Unset: vu == 0 && vp == nil
328 // 2. Untracked: vu != 0 && vp == nil
329 // 3. Tracked: vu != 0 && vp != nil
330 //
331 // Do not set fields manually. Use methods instead.
332 // Extend this type with additional methods if needed.
333 type maybeTraceablePtr struct {
334 vp unsafe.Pointer // For liveness only.
335 vu uintptr // Source of truth.
336 }
337
338 // untrack unsets the pointer but preserves the address.
339 // This is used to hide the pointer from the GC.
340 //
341 //go:nosplit
342 func (p *maybeTraceablePtr) setUntraceable() {
343 p.vp = nil
344 }
345
346 // setTraceable resets the pointer to the stored address.
347 // This is used to make the pointer visible to the GC.
348 //
349 //go:nosplit
350 func (p *maybeTraceablePtr) setTraceable() {
351 p.vp = unsafe.Pointer(p.vu)
352 }
353
354 // set sets the pointer to the data element and updates the address.
355 //
356 //go:nosplit
357 func (p *maybeTraceablePtr) set(v unsafe.Pointer) {
358 p.vp = v
359 p.vu = uintptr(v)
360 }
361
362 // get retrieves the pointer to the data element.
363 //
364 //go:nosplit
365 func (p *maybeTraceablePtr) get() unsafe.Pointer {
366 return unsafe.Pointer(p.vu)
367 }
368
369 // uintptr returns the uintptr address of the pointer.
370 //
371 //go:nosplit
372 func (p *maybeTraceablePtr) uintptr() uintptr {
373 return p.vu
374 }
375
376 // maybeTraceableChan extends conditionally trackable pointers (maybeTraceablePtr)
377 // to track hchan pointers.
378 //
379 // Do not set fields manually. Use methods instead.
380 type maybeTraceableChan struct {
381 maybeTraceablePtr
382 }
383
384 //go:nosplit
385 func (p *maybeTraceableChan) set(c *hchan) {
386 p.maybeTraceablePtr.set(unsafe.Pointer(c))
387 }
388
389 //go:nosplit
390 func (p *maybeTraceableChan) get() *hchan {
391 return (*hchan)(p.maybeTraceablePtr.get())
392 }
393
394 // sudog (pseudo-g) represents a g in a wait list, such as for sending/receiving
395 // on a channel.
396 //
397 // sudog is necessary because the g ↔ synchronization object relation
398 // is many-to-many. A g can be on many wait lists, so there may be
399 // many sudogs for one g; and many gs may be waiting on the same
400 // synchronization object, so there may be many sudogs for one object.
401 //
402 // sudogs are allocated from a special pool. Use acquireSudog and
403 // releaseSudog to allocate and free them.
404 type sudog struct {
405 // The following fields are protected by the hchan.lock of the
406 // channel this sudog is blocking on. shrinkstack depends on
407 // this for sudogs involved in channel ops.
408
409 g *g
410
411 next *sudog
412 prev *sudog
413
414 elem maybeTraceablePtr // data element (may point to stack)
415
416 // The following fields are never accessed concurrently.
417 // For channels, waitlink is only accessed by g.
418 // For semaphores, all fields (including the ones above)
419 // are only accessed when holding a semaRoot lock.
420
421 acquiretime int64
422 releasetime int64
423 ticket uint32
424
425 // isSelect indicates g is participating in a select, so
426 // g.selectDone must be CAS'd to win the wake-up race.
427 isSelect bool
428
429 // success indicates whether communication over channel c
430 // succeeded. It is true if the goroutine was awoken because a
431 // value was delivered over channel c, and false if awoken
432 // because c was closed.
433 success bool
434
435 // waiters is a count of semaRoot waiting list other than head of list,
436 // clamped to a uint16 to fit in unused space.
437 // Only meaningful at the head of the list.
438 // (If we wanted to be overly clever, we could store a high 16 bits
439 // in the second entry in the list.)
440 waiters uint16
441
442 parent *sudog // semaRoot binary tree
443 waitlink *sudog // g.waiting list or semaRoot
444 waittail *sudog // semaRoot
445 c maybeTraceableChan // channel
446 }
447
448 type libcall struct {
449 fn uintptr
450 n uintptr // number of parameters
451 args uintptr // parameters
452 r1 uintptr // return values
453 r2 uintptr
454 err uintptr // error number
455 }
456
457 // Stack describes a Go execution stack.
458 // The bounds of the stack are exactly [lo, hi),
459 // with no implicit data structures on either side.
460 type stack struct {
461 lo uintptr
462 hi uintptr
463 }
464
465 // heldLockInfo gives info on a held lock and the rank of that lock
466 type heldLockInfo struct {
467 lockAddr uintptr
468 rank lockRank
469 }
470
471 type g struct {
472 // Stack parameters.
473 // stack describes the actual stack memory: [stack.lo, stack.hi).
474 // stackguard0 is the stack pointer compared in the Go stack growth prologue.
475 // It is stack.lo+StackGuard normally, but can be StackPreempt to trigger a preemption.
476 // stackguard1 is the stack pointer compared in the //go:systemstack stack growth prologue.
477 // It is stack.lo+StackGuard on g0 and gsignal stacks.
478 // It is ~0 on other goroutine stacks, to trigger a call to morestackc (and crash).
479 stack stack // offset known to runtime/cgo
480 stackguard0 uintptr // offset known to cmd/internal/obj/*
481 stackguard1 uintptr // offset known to cmd/internal/obj/*
482
483 _panic *_panic // innermost panic
484 _defer *_defer // innermost defer
485 m *m // current m
486 sched gobuf
487 syscallsp uintptr // if status==Gsyscall, syscallsp = sched.sp to use during gc
488 syscallpc uintptr // if status==Gsyscall, syscallpc = sched.pc to use during gc
489 syscallbp uintptr // if status==Gsyscall, syscallbp = sched.bp to use in fpTraceback
490 stktopsp uintptr // expected sp at top of stack, to check in traceback
491 // param is a generic pointer parameter field used to pass
492 // values in particular contexts where other storage for the
493 // parameter would be difficult to find. It is currently used
494 // in four ways:
495 // 1. When a channel operation wakes up a blocked goroutine, it sets param to
496 // point to the sudog of the completed blocking operation.
497 // 2. By gcAssistAlloc1 to signal back to its caller that the goroutine completed
498 // the GC cycle. It is unsafe to do so in any other way, because the goroutine's
499 // stack may have moved in the meantime.
500 // 3. By debugCallWrap to pass parameters to a new goroutine because allocating a
501 // closure in the runtime is forbidden.
502 // 4. When a panic is recovered and control returns to the respective frame,
503 // param may point to a savedOpenDeferState.
504 param unsafe.Pointer
505 atomicstatus atomic.Uint32
506 stackLock uint32 // sigprof/scang lock; TODO: fold in to atomicstatus
507 goid uint64
508 schedlink guintptr
509 waitsince int64 // approx time when the g become blocked
510 waitreason waitReason // if status==Gwaiting
511
512 preempt bool // preemption signal, duplicates stackguard0 = stackpreempt
513 preemptStop bool // transition to _Gpreempted on preemption; otherwise, just deschedule
514 preemptShrink bool // shrink stack at synchronous safe point
515
516 // asyncSafePoint is set if g is stopped at an asynchronous
517 // safe point. This means there are frames on the stack
518 // without precise pointer information.
519 asyncSafePoint bool
520
521 paniconfault bool // panic (instead of crash) on unexpected fault address
522 gcscandone bool // g has scanned stack; protected by _Gscan bit in status
523 throwsplit bool // must not split stack
524 // activeStackChans indicates that there are unlocked channels
525 // pointing into this goroutine's stack. If true, stack
526 // copying needs to acquire channel locks to protect these
527 // areas of the stack.
528 activeStackChans bool
529 // parkingOnChan indicates that the goroutine is about to
530 // park on a chansend or chanrecv. Used to signal an unsafe point
531 // for stack shrinking.
532 parkingOnChan atomic.Bool
533 // inMarkAssist indicates whether the goroutine is in mark assist.
534 // Used by the execution tracer.
535 inMarkAssist bool
536 coroexit bool // argument to coroswitch_m
537
538 raceignore int8 // ignore race detection events
539 nocgocallback bool // whether disable callback from C
540 tracking bool // whether we're tracking this G for sched latency statistics
541 trackingSeq uint8 // used to decide whether to track this G
542 trackingStamp int64 // timestamp of when the G last started being tracked
543 runnableTime int64 // the amount of time spent runnable, cleared when running, only used when tracking
544 lockedm muintptr
545 fipsIndicator uint8
546 fipsOnlyBypass bool
547 ditWanted bool // set if g wants to be executed with DIT enabled
548 syncSafePoint bool // set if g is stopped at a synchronous safe point.
549 runningCleanups atomic.Bool
550 sig uint32
551 secret int32 // current nesting of runtime/secret.Do calls.
552 writebuf []byte
553 sigcode0 uintptr
554 sigcode1 uintptr
555 sigpc uintptr
556 parentGoid uint64 // goid of goroutine that created this goroutine
557 gopc uintptr // pc of go statement that created this goroutine
558 ancestors *[]ancestorInfo // ancestor information goroutine(s) that created this goroutine (only used if debug.tracebackancestors)
559 startpc uintptr // pc of goroutine function
560 racectx uintptr
561 waiting *sudog // sudog structures this g is waiting on (that have a valid elem ptr); in lock order
562 cgoCtxt []uintptr // cgo traceback context
563 labels unsafe.Pointer // profiler labels
564 timer *timer // cached timer for time.Sleep
565 sleepWhen int64 // when to sleep until
566 selectDone atomic.Uint32 // are we participating in a select and did someone win the race?
567
568 // goroutineProfiled indicates the status of this goroutine's stack for the
569 // current in-progress goroutine profile
570 goroutineProfiled goroutineProfileStateHolder
571
572 coroarg *coro // argument during coroutine transfers
573 bubble *synctestBubble
574
575 // xRegs stores the extended register state if this G has been
576 // asynchronously preempted.
577 xRegs xRegPerG
578
579 // Per-G tracer state.
580 trace gTraceState
581
582 // Per-G GC state
583
584 // gcAssistBytes is this G's GC assist credit in terms of
585 // bytes allocated. If this is positive, then the G has credit
586 // to allocate gcAssistBytes bytes without assisting. If this
587 // is negative, then the G must correct this by performing
588 // scan work. We track this in bytes to make it fast to update
589 // and check for debt in the malloc hot path. The assist ratio
590 // determines how this corresponds to scan work debt.
591 gcAssistBytes int64
592
593 // valgrindStackID is used to track what memory is used for stacks when a program is
594 // built with the "valgrind" build tag, otherwise it is unused.
595 valgrindStackID uintptr
596 }
597
598 // gTrackingPeriod is the number of transitions out of _Grunning between
599 // latency tracking runs.
600 const gTrackingPeriod = 8
601
602 const (
603 // tlsSlots is the number of pointer-sized slots reserved for TLS on some platforms,
604 // like Windows.
605 tlsSlots = 6
606 tlsSize = tlsSlots * goarch.PtrSize
607 )
608
609 // Values for m.freeWait.
610 const (
611 freeMStack = 0 // M done, free stack and reference.
612 freeMRef = 1 // M done, free reference.
613 freeMWait = 2 // M still in use.
614 )
615
616 type m struct {
617 g0 *g // goroutine with scheduling stack
618 morebuf gobuf // gobuf arg to morestack
619 divmod uint32 // div/mod denominator for arm - known to liblink (cmd/internal/obj/arm/obj5.go)
620
621 // Fields whose offsets are not known to debuggers.
622
623 procid uint64 // for debuggers, but offset not hard-coded
624 gsignal *g // signal-handling g
625 goSigStack gsignalStack // Go-allocated signal handling stack
626 sigmask sigset // storage for saved signal mask
627 tls [tlsSlots]uintptr // thread-local storage (for x86 extern register)
628 mstartfn func()
629 curg *g // current running goroutine
630 caughtsig guintptr // goroutine running during fatal signal
631
632 // Indicates whether we've received a signal while
633 // running in secret mode.
634 signalSecret bool
635
636 // p is the currently attached P for executing Go code, nil if not executing user Go code.
637 //
638 // A non-nil p implies exclusive ownership of the P, unless curg is in _Gsyscall.
639 // In _Gsyscall the scheduler may mutate this instead. The point of synchronization
640 // is the _Gscan bit on curg's status. The scheduler must arrange to prevent curg
641 // from transitioning out of _Gsyscall if it intends to mutate p.
642 p puintptr
643
644 nextp puintptr // The next P to install before executing. Implies exclusive ownership of this P.
645 oldp puintptr // The P that was attached before executing a syscall.
646 id int64
647 mallocing int32
648 throwing throwType
649 preemptoff string // if != "", keep curg running on this m
650 locks int32
651 dying int32
652 profilehz int32
653 spinning bool // m is out of work and is actively looking for work
654 blocked bool // m is blocked on a note
655 newSigstack bool // minit on C thread called sigaltstack
656 printlock int8
657 incgo bool // m is executing a cgo call
658 isextra bool // m is an extra m
659 isExtraInC bool // m is an extra m that does not have any Go frames
660 isExtraInSig bool // m is an extra m in a signal handler
661 freeWait atomic.Uint32 // Whether it is safe to free g0 and delete m (one of freeMRef, freeMStack, freeMWait)
662 needextram bool
663 g0StackAccurate bool // whether the g0 stack has accurate bounds
664 traceback uint8
665 allpSnapshot []*p // Snapshot of allp for use after dropping P in findRunnable, nil otherwise.
666 ncgocall uint64 // number of cgo calls in total
667 ncgo int32 // number of cgo calls currently in progress
668 cgoCallersUse atomic.Uint32 // if non-zero, cgoCallers in use temporarily
669 cgoCallers *cgoCallers // cgo traceback if crashing in cgo call
670 park note
671 alllink *m // on allm
672 schedlink muintptr
673 idleNode listNodeManual
674 lockedg guintptr
675 createstack [32]uintptr // stack that created this thread, it's used for StackRecord.Stack0, so it must align with it.
676 lockedExt uint32 // tracking for external LockOSThread
677 lockedInt uint32 // tracking for internal lockOSThread
678 mWaitList mWaitList // list of runtime lock waiters
679 ditEnabled bool // set if DIT is currently enabled on this M
680
681 mLockProfile mLockProfile // fields relating to runtime.lock contention
682 profStack []uintptr // used for memory/block/mutex stack traces
683
684 // wait* are used to carry arguments from gopark into park_m, because
685 // there's no stack to put them on. That is their sole purpose.
686 waitunlockf func(*g, unsafe.Pointer) bool
687 waitlock unsafe.Pointer
688 waitTraceSkip int
689 waitTraceBlockReason traceBlockReason
690
691 syscalltick uint32
692 freelink *m // on sched.freem
693 trace mTraceState
694
695 // These are here to avoid using the G stack so the stack can move during the call.
696 libcallpc uintptr // for cpu profiler
697 libcallsp uintptr
698 libcallg guintptr
699 winsyscall winlibcall // stores syscall parameters on windows
700
701 vdsoSP uintptr // SP for traceback while in VDSO call (0 if not in call)
702 vdsoPC uintptr // PC for traceback while in VDSO call
703
704 // preemptGen counts the number of completed preemption
705 // signals. This is used to detect when a preemption is
706 // requested, but fails.
707 preemptGen atomic.Uint32
708
709 // Whether this is a pending preemption signal on this M.
710 signalPending atomic.Uint32
711
712 // pcvalue lookup cache
713 pcvalueCache pcvalueCache
714
715 dlogPerM
716
717 mOS
718
719 chacha8 chacha8rand.State
720 cheaprand uint32
721 cheaprand64 uint64
722
723 // Up to 10 locks held by this m, maintained by the lock ranking code.
724 locksHeldLen int
725 locksHeld [10]heldLockInfo
726
727 // self points this M until mexit clears it to return nil.
728 self mWeakPointer
729 }
730
731 const mRedZoneSize = (16 << 3) * asanenabledBit // redZoneSize(2048)
732
733 type mPadded struct {
734 m
735
736 // Size the runtime.m structure so it fits in the 2048-byte size class, and
737 // not in the next-smallest (1792-byte) size class. That leaves the 11 low
738 // bits of muintptr values available for flags, as required by
739 // lock_spinbit.go.
740 _ [(1 - goarch.IsWasm) * (2048 - mallocHeaderSize - mRedZoneSize - unsafe.Sizeof(m{}))]byte
741 }
742
743 // mWeakPointer is a "weak" pointer to an M. A weak pointer for each M is
744 // available as m.self. Users may copy mWeakPointer arbitrarily, and get will
745 // return the M if it is still live, or nil after mexit.
746 //
747 // The zero value is treated as a nil pointer.
748 //
749 // Note that get may race with M exit. A successful get will keep the m object
750 // alive, but the M itself may be exited and thus not actually usable.
751 type mWeakPointer struct {
752 m *atomic.Pointer[m]
753 }
754
755 func newMWeakPointer(mp *m) mWeakPointer {
756 w := mWeakPointer{m: new(atomic.Pointer[m])}
757 w.m.Store(mp)
758 return w
759 }
760
761 func (w mWeakPointer) get() *m {
762 if w.m == nil {
763 return nil
764 }
765 return w.m.Load()
766 }
767
768 // clear sets the weak pointer to nil. It cannot be used on zero value
769 // mWeakPointers.
770 func (w mWeakPointer) clear() {
771 w.m.Store(nil)
772 }
773
774 type p struct {
775 id int32
776 status uint32 // one of pidle/prunning/...
777 link puintptr
778 schedtick uint32 // incremented on every scheduler call
779 syscalltick uint32 // incremented on every system call
780 sysmontick sysmontick // last tick observed by sysmon
781 m muintptr // back-link to associated m (nil if idle)
782 mcache *mcache
783 pcache pageCache
784 raceprocctx uintptr
785
786 // oldm is the previous m this p ran on.
787 //
788 // We are not associated with this m, so we have no control over its
789 // lifecycle. This value is an m.self object which points to the m
790 // until the m exits.
791 //
792 // Note that this m may be idle, running, or exiting. It should only be
793 // used with mgetSpecific, which will take ownership of the m only if
794 // it is idle.
795 oldm mWeakPointer
796
797 deferpool []*_defer // pool of available defer structs (see panic.go)
798 deferpoolbuf [32]*_defer
799
800 // Cache of goroutine ids, amortizes accesses to runtime·sched.goidgen.
801 goidcache uint64
802 goidcacheend uint64
803
804 // Queue of runnable goroutines. Accessed without lock.
805 runqhead uint32
806 runqtail uint32
807 runq [256]guintptr
808 // runnext, if non-nil, is a runnable G that was ready'd by
809 // the current G and should be run next instead of what's in
810 // runq if there's time remaining in the running G's time
811 // slice. It will inherit the time left in the current time
812 // slice. If a set of goroutines is locked in a
813 // communicate-and-wait pattern, this schedules that set as a
814 // unit and eliminates the (potentially large) scheduling
815 // latency that otherwise arises from adding the ready'd
816 // goroutines to the end of the run queue.
817 //
818 // Note that while other P's may atomically CAS this to zero,
819 // only the owner P can CAS it to a valid G.
820 runnext guintptr
821
822 // Available G's (status == Gdead)
823 gFree gList
824
825 sudogcache []*sudog
826 sudogbuf [128]*sudog
827
828 // Cache of mspan objects from the heap.
829 mspancache struct {
830 // We need an explicit length here because this field is used
831 // in allocation codepaths where write barriers are not allowed,
832 // and eliminating the write barrier/keeping it eliminated from
833 // slice updates is tricky, more so than just managing the length
834 // ourselves.
835 len int
836 buf [128]*mspan
837 }
838
839 // Cache of a single pinner object to reduce allocations from repeated
840 // pinner creation.
841 pinnerCache *pinner
842
843 trace pTraceState
844
845 palloc persistentAlloc // per-P to avoid mutex
846
847 // Per-P GC state
848 gcAssistTime int64 // Nanoseconds in assistAlloc
849 gcFractionalMarkTime atomic.Int64 // Nanoseconds in fractional mark worker
850
851 // limiterEvent tracks events for the GC CPU limiter.
852 limiterEvent limiterEvent
853
854 // gcMarkWorkerMode is the mode for the next mark worker to run in.
855 // That is, this is used to communicate with the worker goroutine
856 // selected for immediate execution by
857 // gcController.findRunnableGCWorker. When scheduling other goroutines,
858 // this field must be set to gcMarkWorkerNotWorker.
859 gcMarkWorkerMode gcMarkWorkerMode
860 // gcMarkWorkerStartTime is the nanotime() at which the most recent
861 // mark worker started.
862 gcMarkWorkerStartTime int64
863
864 // nextGCMarkWorker is the next mark worker to run. This may be set
865 // during start-the-world to assign a worker to this P. The P runs this
866 // worker on the next call to gcController.findRunnableGCWorker. If the
867 // P runs something else or stops, it must release this worker via
868 // gcController.releaseNextGCMarkWorker.
869 //
870 // See comment in gcBgMarkWorker about the lifetime of
871 // gcBgMarkWorkerNode.
872 //
873 // Only accessed by this P or during STW.
874 nextGCMarkWorker *gcBgMarkWorkerNode
875
876 // gcw is this P's GC work buffer cache. The work buffer is
877 // filled by write barriers, drained by mutator assists, and
878 // disposed on certain GC state transitions.
879 gcw gcWork
880
881 // wbBuf is this P's GC write barrier buffer.
882 //
883 // TODO: Consider caching this in the running G.
884 wbBuf wbBuf
885
886 runSafePointFn uint32 // if 1, run sched.safePointFn at next safe point
887
888 // statsSeq is a counter indicating whether this P is currently
889 // writing any stats. Its value is even when not, odd when it is.
890 statsSeq atomic.Uint32
891
892 // Timer heap.
893 timers timers
894
895 // Cleanups.
896 cleanups *cleanupBlock
897 cleanupsQueued uint64 // monotonic count of cleanups queued by this P
898
899 // maxStackScanDelta accumulates the amount of stack space held by
900 // live goroutines (i.e. those eligible for stack scanning).
901 // Flushed to gcController.maxStackScan once maxStackScanSlack
902 // or -maxStackScanSlack is reached.
903 maxStackScanDelta int64
904
905 // gc-time statistics about current goroutines
906 // Note that this differs from maxStackScan in that this
907 // accumulates the actual stack observed to be used at GC time (hi - sp),
908 // not an instantaneous measure of the total stack size that might need
909 // to be scanned (hi - lo).
910 scannedStackSize uint64 // stack size of goroutines scanned by this P
911 scannedStacks uint64 // number of goroutines scanned by this P
912
913 // preempt is set to indicate that this P should be enter the
914 // scheduler ASAP (regardless of what G is running on it).
915 preempt bool
916
917 // gcStopTime is the nanotime timestamp that this P last entered _Pgcstop.
918 gcStopTime int64
919
920 // goroutinesCreated is the total count of goroutines created by this P.
921 goroutinesCreated uint64
922
923 // xRegs is the per-P extended register state used by asynchronous
924 // preemption. This is an empty struct on platforms that don't use extended
925 // register state.
926 xRegs xRegPerP
927
928 // Padding is no longer needed. False sharing is now not a worry because p is large enough
929 // that its size class is an integer multiple of the cache line size (for any of our architectures).
930 }
931
932 type schedt struct {
933 goidgen atomic.Uint64
934 lastpoll atomic.Int64 // time of last network poll, 0 if currently polling
935 pollUntil atomic.Int64 // time to which current poll is sleeping
936 pollingNet atomic.Int32 // 1 if some P doing non-blocking network poll
937
938 lock mutex
939
940 // When increasing nmidle, nmidlelocked, nmsys, or nmfreed, be
941 // sure to call checkdead().
942
943 midle listHeadManual // idle m's waiting for work
944 nmidle int32 // number of idle m's waiting for work
945 nmidlelocked int32 // number of locked m's waiting for work
946 mnext int64 // number of m's that have been created and next M ID
947 maxmcount int32 // maximum number of m's allowed (or die)
948 nmsys int32 // number of system m's not counted for deadlock
949 nmfreed int64 // cumulative number of freed m's
950
951 ngsys atomic.Int32 // number of system goroutines
952 nGsyscallNoP atomic.Int32 // number of goroutines in syscalls without a P but whose M is not isExtraInC
953
954 pidle puintptr // idle p's
955 npidle atomic.Int32
956 nmspinning atomic.Int32 // See "Worker thread parking/unparking" comment in proc.go.
957 needspinning atomic.Uint32 // See "Delicate dance" comment in proc.go. Boolean. Must hold sched.lock to set to 1.
958
959 // Global runnable queue.
960 runq gQueue
961
962 // disable controls selective disabling of the scheduler.
963 //
964 // Use schedEnableUser to control this.
965 //
966 // disable is protected by sched.lock.
967 disable struct {
968 // user disables scheduling of user goroutines.
969 user bool
970 runnable gQueue // pending runnable Gs
971 }
972
973 // Global cache of dead G's.
974 gFree struct {
975 lock mutex
976 stack gList // Gs with stacks
977 noStack gList // Gs without stacks
978 }
979
980 // Central cache of sudog structs.
981 sudoglock mutex
982 sudogcache *sudog
983
984 // Central pool of available defer structs.
985 deferlock mutex
986 deferpool *_defer
987
988 // freem is the list of m's waiting to be freed when their
989 // m.exited is set. Linked through m.freelink.
990 freem *m
991
992 gcwaiting atomic.Bool // gc is waiting to run
993 stopwait int32
994 stopnote note
995 sysmonwait atomic.Bool
996 sysmonnote note
997
998 // safePointFn should be called on each P at the next GC
999 // safepoint if p.runSafePointFn is set.
1000 safePointFn func(*p)
1001 safePointWait int32
1002 safePointNote note
1003
1004 profilehz int32 // cpu profiling rate
1005
1006 procresizetime int64 // nanotime() of last change to gomaxprocs
1007 totaltime int64 // ∫gomaxprocs dt up to procresizetime
1008
1009 customGOMAXPROCS bool // GOMAXPROCS was manually set from the environment or runtime.GOMAXPROCS
1010
1011 // sysmonlock protects sysmon's actions on the runtime.
1012 //
1013 // Acquire and hold this mutex to block sysmon from interacting
1014 // with the rest of the runtime.
1015 sysmonlock mutex
1016
1017 // timeToRun is a distribution of scheduling latencies, defined
1018 // as the sum of time a G spends in the _Grunnable state before
1019 // it transitions to _Grunning.
1020 timeToRun timeHistogram
1021
1022 // idleTime is the total CPU time Ps have "spent" idle.
1023 //
1024 // Reset on each GC cycle.
1025 idleTime atomic.Int64
1026
1027 // totalMutexWaitTime is the sum of time goroutines have spent in _Gwaiting
1028 // with a waitreason of the form waitReasonSync{RW,}Mutex{R,}Lock.
1029 totalMutexWaitTime atomic.Int64
1030
1031 // stwStoppingTimeGC/Other are distributions of stop-the-world stopping
1032 // latencies, defined as the time taken by stopTheWorldWithSema to get
1033 // all Ps to stop. stwStoppingTimeGC covers all GC-related STWs,
1034 // stwStoppingTimeOther covers the others.
1035 stwStoppingTimeGC timeHistogram
1036 stwStoppingTimeOther timeHistogram
1037
1038 // stwTotalTimeGC/Other are distributions of stop-the-world total
1039 // latencies, defined as the total time from stopTheWorldWithSema to
1040 // startTheWorldWithSema. This is a superset of
1041 // stwStoppingTimeGC/Other. stwTotalTimeGC covers all GC-related STWs,
1042 // stwTotalTimeOther covers the others.
1043 stwTotalTimeGC timeHistogram
1044 stwTotalTimeOther timeHistogram
1045
1046 // totalRuntimeLockWaitTime (plus the value of lockWaitTime on each M in
1047 // allm) is the sum of time goroutines have spent in _Grunnable and with an
1048 // M, but waiting for locks within the runtime. This field stores the value
1049 // for Ms that have exited.
1050 totalRuntimeLockWaitTime atomic.Int64
1051
1052 // goroutinesCreated (plus the value of goroutinesCreated on each P in allp)
1053 // is the sum of all goroutines created by the program.
1054 goroutinesCreated atomic.Uint64
1055 }
1056
1057 // Values for the flags field of a sigTabT.
1058 const (
1059 _SigNotify = 1 << iota // let signal.Notify have signal, even if from kernel
1060 _SigKill // if signal.Notify doesn't take it, exit quietly
1061 _SigThrow // if signal.Notify doesn't take it, exit loudly
1062 _SigPanic // if the signal is from the kernel, panic
1063 _SigDefault // if the signal isn't explicitly requested, don't monitor it
1064 _SigGoExit // cause all runtime procs to exit (only used on Plan 9).
1065 _SigSetStack // Don't explicitly install handler, but add SA_ONSTACK to existing libc handler
1066 _SigUnblock // always unblock; see blockableSig
1067 _SigIgn // _SIG_DFL action is to ignore the signal
1068 )
1069
1070 // Layout of in-memory per-function information prepared by linker
1071 // See https://golang.org/s/go12symtab.
1072 // Keep in sync with linker (../cmd/link/internal/ld/pcln.go:/pclntab)
1073 // and with package debug/gosym and with symtab.go in package runtime.
1074 type _func struct {
1075 sys.NotInHeap // Only in static data
1076
1077 entryOff uint32 // start pc, as offset from moduledata.text
1078 nameOff int32 // function name, as index into moduledata.funcnametab.
1079
1080 args int32 // in/out args size
1081 deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any.
1082
1083 pcsp uint32
1084 pcfile uint32
1085 pcln uint32
1086 npcdata uint32
1087 cuOffset uint32 // runtime.cutab offset of this function's CU
1088 startLine int32 // line number of start of function (func keyword/TEXT directive)
1089 funcID abi.FuncID // set for certain special runtime functions
1090 flag abi.FuncFlag
1091 _ [1]byte // pad
1092 nfuncdata uint8 // must be last, must end on a uint32-aligned boundary
1093
1094 // The end of the struct is followed immediately by two variable-length
1095 // arrays that reference the pcdata and funcdata locations for this
1096 // function.
1097
1098 // pcdata contains the offset into moduledata.pctab for the start of
1099 // that index's table. e.g.,
1100 // &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of
1101 // the unsafe point table.
1102 //
1103 // An offset of 0 indicates that there is no table.
1104 //
1105 // pcdata [npcdata]uint32
1106
1107 // funcdata contains the offset past moduledata.gofunc which contains a
1108 // pointer to that index's funcdata. e.g.,
1109 // *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is
1110 // the argument pointer map.
1111 //
1112 // An offset of ^uint32(0) indicates that there is no entry.
1113 //
1114 // funcdata [nfuncdata]uint32
1115 }
1116
1117 // Pseudo-Func that is returned for PCs that occur in inlined code.
1118 // A *Func can be either a *_func or a *funcinl, and they are distinguished
1119 // by the first uintptr.
1120 //
1121 // TODO(austin): Can we merge this with inlinedCall?
1122 type funcinl struct {
1123 ones uint32 // set to ^0 to distinguish from _func
1124 entry uintptr // entry of the real (the "outermost") frame
1125 name string
1126 file string
1127 line int32
1128 startLine int32
1129 }
1130
1131 type itab = abi.ITab
1132
1133 // Lock-free stack node.
1134 // Also known to export_test.go.
1135 type lfnode struct {
1136 next uint64
1137 pushcnt uintptr
1138 }
1139
1140 type forcegcstate struct {
1141 lock mutex
1142 g *g
1143 idle atomic.Bool
1144 }
1145
1146 // A _defer holds an entry on the list of deferred calls.
1147 // If you add a field here, add code to clear it in deferProcStack.
1148 // This struct must match the code in cmd/compile/internal/ssagen/ssa.go:deferstruct
1149 // and cmd/compile/internal/ssagen/ssa.go:(*state).call.
1150 // Some defers will be allocated on the stack and some on the heap.
1151 // All defers are logically part of the stack, so write barriers to
1152 // initialize them are not required. All defers must be manually scanned,
1153 // and for heap defers, marked.
1154 type _defer struct {
1155 heap bool
1156 rangefunc bool // true for rangefunc list
1157 sp uintptr // sp at time of defer
1158 pc uintptr // pc at time of defer
1159 fn func() // can be nil for open-coded defers
1160 link *_defer // next defer on G; can point to either heap or stack!
1161
1162 // If rangefunc is true, *head is the head of the atomic linked list
1163 // during a range-over-func execution.
1164 head *atomic.Pointer[_defer]
1165 }
1166
1167 // A _panic holds information about an active panic.
1168 //
1169 // A _panic value must only ever live on the stack.
1170 //
1171 // The gopanicFP and link fields are stack pointers, but don't need special
1172 // handling during stack growth: because they are pointer-typed and
1173 // _panic values only live on the stack, regular stack pointer
1174 // adjustment takes care of them.
1175 type _panic struct {
1176 arg any // argument to panic
1177 link *_panic // link to earlier panic
1178
1179 // startPC and startSP track where _panic.start was called.
1180 // (These are the SP and PC of the gopanic frame itself.)
1181 startPC uintptr
1182 startSP unsafe.Pointer
1183
1184 // The current stack frame that we're running deferred calls for.
1185 pc uintptr
1186 sp unsafe.Pointer
1187 fp unsafe.Pointer
1188
1189 // retpc stores the PC where the panic should jump back to, if the
1190 // function last returned by _panic.nextDefer() recovers the panic.
1191 retpc uintptr
1192
1193 // Extra state for handling open-coded defers.
1194 deferBitsPtr *uint8
1195 slotsPtr unsafe.Pointer
1196
1197 recovered bool // whether this panic has been recovered
1198 repanicked bool // whether this panic repanicked
1199 goexit bool
1200 deferreturn bool
1201 }
1202
1203 // savedOpenDeferState tracks the extra state from _panic that's
1204 // necessary for deferreturn to pick up where gopanic left off,
1205 // without needing to unwind the stack.
1206 type savedOpenDeferState struct {
1207 retpc uintptr
1208 deferBitsOffset uintptr
1209 slotsOffset uintptr
1210 }
1211
1212 // ancestorInfo records details of where a goroutine was started.
1213 type ancestorInfo struct {
1214 pcs []uintptr // pcs from the stack of this goroutine
1215 goid uint64 // goroutine id of this goroutine; original goroutine possibly dead
1216 gopc uintptr // pc of go statement that created this goroutine
1217 }
1218
1219 // A waitReason explains why a goroutine has been stopped.
1220 // See gopark. Do not re-use waitReasons, add new ones.
1221 type waitReason uint8
1222
1223 const (
1224 waitReasonZero waitReason = iota // ""
1225 waitReasonGCAssistMarking // "GC assist marking"
1226 waitReasonIOWait // "IO wait"
1227 waitReasonDumpingHeap // "dumping heap"
1228 waitReasonGarbageCollection // "garbage collection"
1229 waitReasonGarbageCollectionScan // "garbage collection scan"
1230 waitReasonPanicWait // "panicwait"
1231 waitReasonGCAssistWait // "GC assist wait"
1232 waitReasonGCSweepWait // "GC sweep wait"
1233 waitReasonGCScavengeWait // "GC scavenge wait"
1234 waitReasonFinalizerWait // "finalizer wait"
1235 waitReasonForceGCIdle // "force gc (idle)"
1236 waitReasonUpdateGOMAXPROCSIdle // "GOMAXPROCS updater (idle)"
1237 waitReasonSemacquire // "semacquire"
1238 waitReasonSleep // "sleep"
1239 waitReasonChanReceiveNilChan // "chan receive (nil chan)"
1240 waitReasonChanSendNilChan // "chan send (nil chan)"
1241 waitReasonSelectNoCases // "select (no cases)"
1242 waitReasonSelect // "select"
1243 waitReasonChanReceive // "chan receive"
1244 waitReasonChanSend // "chan send"
1245 waitReasonSyncCondWait // "sync.Cond.Wait"
1246 waitReasonSyncMutexLock // "sync.Mutex.Lock"
1247 waitReasonSyncRWMutexRLock // "sync.RWMutex.RLock"
1248 waitReasonSyncRWMutexLock // "sync.RWMutex.Lock"
1249 waitReasonSyncWaitGroupWait // "sync.WaitGroup.Wait"
1250 waitReasonTraceReaderBlocked // "trace reader (blocked)"
1251 waitReasonWaitForGCCycle // "wait for GC cycle"
1252 waitReasonGCWorkerIdle // "GC worker (idle)"
1253 waitReasonGCWorkerActive // "GC worker (active)"
1254 waitReasonPreempted // "preempted"
1255 waitReasonDebugCall // "debug call"
1256 waitReasonGCMarkTermination // "GC mark termination"
1257 waitReasonStoppingTheWorld // "stopping the world"
1258 waitReasonFlushProcCaches // "flushing proc caches"
1259 waitReasonTraceGoroutineStatus // "trace goroutine status"
1260 waitReasonTraceProcStatus // "trace proc status"
1261 waitReasonPageTraceFlush // "page trace flush"
1262 waitReasonCoroutine // "coroutine"
1263 waitReasonGCWeakToStrongWait // "GC weak to strong wait"
1264 waitReasonSynctestRun // "synctest.Run"
1265 waitReasonSynctestWait // "synctest.Wait"
1266 waitReasonSynctestChanReceive // "chan receive (durable)"
1267 waitReasonSynctestChanSend // "chan send (durable)"
1268 waitReasonSynctestSelect // "select (durable)"
1269 waitReasonSynctestWaitGroupWait // "sync.WaitGroup.Wait (durable)"
1270 waitReasonCleanupWait // "cleanup wait"
1271 )
1272
1273 var waitReasonStrings = [...]string{
1274 waitReasonZero: "",
1275 waitReasonGCAssistMarking: "GC assist marking",
1276 waitReasonIOWait: "IO wait",
1277 waitReasonChanReceiveNilChan: "chan receive (nil chan)",
1278 waitReasonChanSendNilChan: "chan send (nil chan)",
1279 waitReasonDumpingHeap: "dumping heap",
1280 waitReasonGarbageCollection: "garbage collection",
1281 waitReasonGarbageCollectionScan: "garbage collection scan",
1282 waitReasonPanicWait: "panicwait",
1283 waitReasonSelect: "select",
1284 waitReasonSelectNoCases: "select (no cases)",
1285 waitReasonGCAssistWait: "GC assist wait",
1286 waitReasonGCSweepWait: "GC sweep wait",
1287 waitReasonGCScavengeWait: "GC scavenge wait",
1288 waitReasonChanReceive: "chan receive",
1289 waitReasonChanSend: "chan send",
1290 waitReasonFinalizerWait: "finalizer wait",
1291 waitReasonForceGCIdle: "force gc (idle)",
1292 waitReasonUpdateGOMAXPROCSIdle: "GOMAXPROCS updater (idle)",
1293 waitReasonSemacquire: "semacquire",
1294 waitReasonSleep: "sleep",
1295 waitReasonSyncCondWait: "sync.Cond.Wait",
1296 waitReasonSyncMutexLock: "sync.Mutex.Lock",
1297 waitReasonSyncRWMutexRLock: "sync.RWMutex.RLock",
1298 waitReasonSyncRWMutexLock: "sync.RWMutex.Lock",
1299 waitReasonSyncWaitGroupWait: "sync.WaitGroup.Wait",
1300 waitReasonTraceReaderBlocked: "trace reader (blocked)",
1301 waitReasonWaitForGCCycle: "wait for GC cycle",
1302 waitReasonGCWorkerIdle: "GC worker (idle)",
1303 waitReasonGCWorkerActive: "GC worker (active)",
1304 waitReasonPreempted: "preempted",
1305 waitReasonDebugCall: "debug call",
1306 waitReasonGCMarkTermination: "GC mark termination",
1307 waitReasonStoppingTheWorld: "stopping the world",
1308 waitReasonFlushProcCaches: "flushing proc caches",
1309 waitReasonTraceGoroutineStatus: "trace goroutine status",
1310 waitReasonTraceProcStatus: "trace proc status",
1311 waitReasonPageTraceFlush: "page trace flush",
1312 waitReasonCoroutine: "coroutine",
1313 waitReasonGCWeakToStrongWait: "GC weak to strong wait",
1314 waitReasonSynctestRun: "synctest.Run",
1315 waitReasonSynctestWait: "synctest.Wait",
1316 waitReasonSynctestChanReceive: "chan receive (durable)",
1317 waitReasonSynctestChanSend: "chan send (durable)",
1318 waitReasonSynctestSelect: "select (durable)",
1319 waitReasonSynctestWaitGroupWait: "sync.WaitGroup.Wait (durable)",
1320 waitReasonCleanupWait: "cleanup wait",
1321 }
1322
1323 func (w waitReason) String() string {
1324 if w < 0 || w >= waitReason(len(waitReasonStrings)) {
1325 return "unknown wait reason"
1326 }
1327 return waitReasonStrings[w]
1328 }
1329
1330 // isMutexWait returns true if the goroutine is blocked because of
1331 // sync.Mutex.Lock or sync.RWMutex.[R]Lock.
1332 //
1333 //go:nosplit
1334 func (w waitReason) isMutexWait() bool {
1335 return w == waitReasonSyncMutexLock ||
1336 w == waitReasonSyncRWMutexRLock ||
1337 w == waitReasonSyncRWMutexLock
1338 }
1339
1340 // isSyncWait returns true if the goroutine is blocked because of
1341 // sync library primitive operations.
1342 //
1343 //go:nosplit
1344 func (w waitReason) isSyncWait() bool {
1345 return waitReasonSyncCondWait <= w && w <= waitReasonSyncWaitGroupWait
1346 }
1347
1348 // isChanWait is true if the goroutine is blocked because of non-nil
1349 // channel operations or a select statement with at least one case.
1350 //
1351 //go:nosplit
1352 func (w waitReason) isChanWait() bool {
1353 return w == waitReasonSelect ||
1354 w == waitReasonChanReceive ||
1355 w == waitReasonChanSend
1356 }
1357
1358 func (w waitReason) isWaitingForSuspendG() bool {
1359 return isWaitingForSuspendG[w]
1360 }
1361
1362 // isWaitingForSuspendG indicates that a goroutine is only entering _Gwaiting and
1363 // setting a waitReason because it needs to be able to let the suspendG
1364 // (used by the GC and the execution tracer) take ownership of its stack.
1365 // The G is always actually executing on the system stack in these cases.
1366 //
1367 // TODO(mknyszek): Consider replacing this with a new dedicated G status.
1368 var isWaitingForSuspendG = [len(waitReasonStrings)]bool{
1369 waitReasonStoppingTheWorld: true,
1370 waitReasonGCMarkTermination: true,
1371 waitReasonGarbageCollection: true,
1372 waitReasonGarbageCollectionScan: true,
1373 waitReasonTraceGoroutineStatus: true,
1374 waitReasonTraceProcStatus: true,
1375 waitReasonPageTraceFlush: true,
1376 waitReasonGCAssistMarking: true,
1377 waitReasonGCWorkerActive: true,
1378 waitReasonFlushProcCaches: true,
1379 }
1380
1381 func (w waitReason) isIdleInSynctest() bool {
1382 return isIdleInSynctest[w]
1383 }
1384
1385 // isIdleInSynctest indicates that a goroutine is considered idle by synctest.Wait.
1386 var isIdleInSynctest = [len(waitReasonStrings)]bool{
1387 waitReasonChanReceiveNilChan: true,
1388 waitReasonChanSendNilChan: true,
1389 waitReasonSelectNoCases: true,
1390 waitReasonSleep: true,
1391 waitReasonSyncCondWait: true,
1392 waitReasonSynctestWaitGroupWait: true,
1393 waitReasonCoroutine: true,
1394 waitReasonSynctestRun: true,
1395 waitReasonSynctestWait: true,
1396 waitReasonSynctestChanReceive: true,
1397 waitReasonSynctestChanSend: true,
1398 waitReasonSynctestSelect: true,
1399 }
1400
1401 var (
1402 // Linked-list of all Ms. Written under sched.lock, read atomically.
1403 allm *m
1404
1405 gomaxprocs int32
1406 numCPUStartup int32
1407 forcegc forcegcstate
1408 sched schedt
1409 newprocs int32
1410 )
1411
1412 var (
1413 // allpLock protects P-less reads and size changes of allp, idlepMask,
1414 // and timerpMask, and all writes to allp.
1415 allpLock mutex
1416
1417 // len(allp) == gomaxprocs; may change at safe points, otherwise
1418 // immutable.
1419 allp []*p
1420
1421 // Bitmask of Ps in _Pidle list, one bit per P. Reads and writes must
1422 // be atomic. Length may change at safe points.
1423 //
1424 // Each P must update only its own bit. In order to maintain
1425 // consistency, a P going idle must set the idle mask simultaneously with
1426 // updates to the idle P list under the sched.lock, otherwise a racing
1427 // pidleget may clear the mask before pidleput sets the mask,
1428 // corrupting the bitmap.
1429 //
1430 // N.B., procresize takes ownership of all Ps in stopTheWorldWithSema.
1431 idlepMask pMask
1432
1433 // Bitmask of Ps that may have a timer, one bit per P. Reads and writes
1434 // must be atomic. Length may change at safe points.
1435 //
1436 // Ideally, the timer mask would be kept immediately consistent on any timer
1437 // operations. Unfortunately, updating a shared global data structure in the
1438 // timer hot path adds too much overhead in applications frequently switching
1439 // between no timers and some timers.
1440 //
1441 // As a compromise, the timer mask is updated only on pidleget / pidleput. A
1442 // running P (returned by pidleget) may add a timer at any time, so its mask
1443 // must be set. An idle P (passed to pidleput) cannot add new timers while
1444 // idle, so if it has no timers at that time, its mask may be cleared.
1445 //
1446 // Thus, we get the following effects on timer-stealing in findRunnable:
1447 //
1448 // - Idle Ps with no timers when they go idle are never checked in findRunnable
1449 // (for work- or timer-stealing; this is the ideal case).
1450 // - Running Ps must always be checked.
1451 // - Idle Ps whose timers are stolen must continue to be checked until they run
1452 // again, even after timer expiration.
1453 //
1454 // When the P starts running again, the mask should be set, as a timer may be
1455 // added at any time.
1456 //
1457 // TODO(prattmic): Additional targeted updates may improve the above cases.
1458 // e.g., updating the mask when stealing a timer.
1459 timerpMask pMask
1460 )
1461
1462 // goarmsoftfp is used by runtime/cgo assembly.
1463 //
1464 //go:linkname goarmsoftfp
1465
1466 var (
1467 // Pool of GC parked background workers. Entries are type
1468 // *gcBgMarkWorkerNode.
1469 gcBgMarkWorkerPool lfstack
1470
1471 // Total number of gcBgMarkWorker goroutines. Protected by worldsema.
1472 gcBgMarkWorkerCount int32
1473
1474 // Information about what cpu features are available.
1475 // Packages outside the runtime should not use these
1476 // as they are not an external api.
1477 // Set on startup in asm_{386,amd64}.s
1478 processorVersionInfo uint32
1479 isIntel bool
1480 )
1481
1482 // set by cmd/link on arm systems
1483 // accessed using linkname by internal/runtime/atomic.
1484 //
1485 // goarm should be an internal detail,
1486 // but widely used packages access it using linkname.
1487 // Notable members of the hall of shame include:
1488 // - github.com/creativeprojects/go-selfupdate
1489 //
1490 // Do not remove or change the type signature.
1491 // See go.dev/issue/67401.
1492 //
1493 //go:linkname goarm
1494 var (
1495 goarm uint8
1496 goarmsoftfp uint8
1497 )
1498
1499 // Set by the linker so the runtime can determine the buildmode.
1500 var (
1501 islibrary bool // -buildmode=c-shared
1502 isarchive bool // -buildmode=c-archive
1503 )
1504
1505 // Must agree with internal/buildcfg.FramePointerEnabled.
1506 const framepointer_enabled = GOARCH == "amd64" || GOARCH == "arm64"
1507
1508 // getcallerfp returns the frame pointer of the caller of the caller
1509 // of this function.
1510 //
1511 //go:nosplit
1512 //go:noinline
1513 func getcallerfp() uintptr {
1514 fp := getfp() // This frame's FP.
1515 if fp != 0 {
1516 fp = *(*uintptr)(unsafe.Pointer(fp)) // The caller's FP.
1517 fp = *(*uintptr)(unsafe.Pointer(fp)) // The caller's caller's FP.
1518 }
1519 return fp
1520 }
1521
View as plain text