Source file
src/runtime/stack.go
1 // Copyright 2013 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 package runtime
6
7 import (
8 "internal/abi"
9 "internal/cpu"
10 "internal/goarch"
11 "internal/goexperiment"
12 "internal/goos"
13 "internal/runtime/atomic"
14 "internal/runtime/gc"
15 "internal/runtime/sys"
16 "math/bits"
17 "unsafe"
18 )
19
20 /*
21 Stack layout parameters.
22 Included both by runtime (compiled via 6c) and linkers (compiled via gcc).
23
24 The per-goroutine g->stackguard is set to point StackGuard bytes
25 above the bottom of the stack. Each function compares its stack
26 pointer against g->stackguard to check for overflow. To cut one
27 instruction from the check sequence for functions with tiny frames,
28 the stack is allowed to protrude StackSmall bytes below the stack
29 guard. Functions with large frames don't bother with the check and
30 always call morestack. The sequences are (for amd64, others are
31 similar):
32
33 guard = g->stackguard
34 frame = function's stack frame size
35 argsize = size of function arguments (call + return)
36
37 stack frame size <= StackSmall:
38 CMPQ guard, SP
39 JHI 3(PC)
40 MOVQ m->morearg, $(argsize << 32)
41 CALL morestack(SB)
42
43 stack frame size > StackSmall but < StackBig
44 LEAQ (frame-StackSmall)(SP), R0
45 CMPQ guard, R0
46 JHI 3(PC)
47 MOVQ m->morearg, $(argsize << 32)
48 CALL morestack(SB)
49
50 stack frame size >= StackBig:
51 MOVQ m->morearg, $((argsize << 32) | frame)
52 CALL morestack(SB)
53
54 The bottom StackGuard - StackSmall bytes are important: there has
55 to be enough room to execute functions that refuse to check for
56 stack overflow, either because they need to be adjacent to the
57 actual caller's frame (deferproc) or because they handle the imminent
58 stack overflow (morestack).
59
60 For example, deferproc might call malloc, which does one of the
61 above checks (without allocating a full frame), which might trigger
62 a call to morestack. This sequence needs to fit in the bottom
63 section of the stack. On amd64, morestack's frame is 40 bytes, and
64 deferproc's frame is 56 bytes. That fits well within the
65 StackGuard - StackSmall bytes at the bottom.
66 The linkers explore all possible call traces involving non-splitting
67 functions to make sure that this limit cannot be violated.
68 */
69
70 const (
71 // stackSystem is a number of additional bytes to add
72 // to each stack below the usual guard area for OS-specific
73 // purposes like signal handling. Used on Windows, Plan 9,
74 // and iOS because they do not use a separate stack.
75 stackSystem = goos.IsWindows*4096 + goos.IsPlan9*512 + goos.IsIos*goarch.IsArm64*1024
76
77 // The minimum size of stack used by Go code
78 stackMin = 2048
79
80 // The minimum stack size to allocate.
81 // The hackery here rounds fixedStack0 up to a power of 2.
82 fixedStack0 = stackMin + stackSystem
83 fixedStack1 = fixedStack0 - 1
84 fixedStack2 = fixedStack1 | (fixedStack1 >> 1)
85 fixedStack3 = fixedStack2 | (fixedStack2 >> 2)
86 fixedStack4 = fixedStack3 | (fixedStack3 >> 4)
87 fixedStack5 = fixedStack4 | (fixedStack4 >> 8)
88 fixedStack6 = fixedStack5 | (fixedStack5 >> 16)
89 fixedStack = fixedStack6 + 1
90
91 // stackNosplit is the maximum number of bytes that a chain of NOSPLIT
92 // functions can use.
93 // This arithmetic must match that in cmd/internal/objabi/stack.go:StackNosplit.
94 stackNosplit = abi.StackNosplitBase * sys.StackGuardMultiplier
95
96 // The stack guard is a pointer this many bytes above the
97 // bottom of the stack.
98 //
99 // The guard leaves enough room for a stackNosplit chain of NOSPLIT calls
100 // plus one stackSmall frame plus stackSystem bytes for the OS.
101 // This arithmetic must match that in cmd/internal/objabi/stack.go:StackLimit.
102 stackGuard = stackNosplit + stackSystem + abi.StackSmall
103 )
104
105 const (
106 // stackDebug == 0: no logging
107 // == 1: logging of per-stack operations
108 // == 2: logging of per-frame operations
109 // == 3: logging of per-word updates
110 // == 4: logging of per-word reads
111 stackDebug = 0
112 stackFromSystem = 0 // allocate stacks from system memory instead of the heap
113 stackFaultOnFree = 0 // old stacks are mapped noaccess to detect use after free
114 stackNoCache = 0 // disable per-P small stack caches
115
116 // check the BP links during traceback.
117 debugCheckBP = false
118 )
119
120 var (
121 stackPoisonCopy = 0 // fill stack that should not be accessed with garbage, to detect bad dereferences during copy
122 )
123
124 const (
125 uintptrMask = 1<<(8*goarch.PtrSize) - 1
126
127 // The values below can be stored to g.stackguard0 to force
128 // the next stack check to fail.
129 // These are all larger than any real SP.
130
131 // Goroutine preemption request.
132 // 0xfffffade in hex.
133 stackPreempt = uintptrMask & -1314
134
135 // Thread is forking. Causes a split stack check failure.
136 // 0xfffffb2e in hex.
137 stackFork = uintptrMask & -1234
138
139 // Force a stack movement. Used for debugging.
140 // 0xfffffeed in hex.
141 stackForceMove = uintptrMask & -275
142
143 // stackPoisonMin is the lowest allowed stack poison value.
144 stackPoisonMin = uintptrMask & -4096
145 )
146
147 // Global pool of spans that have free stacks.
148 // Stacks are assigned an order according to size.
149 //
150 // order = log_2(size/FixedStack)
151 //
152 // There is a free list for each order.
153 var stackpool [_NumStackOrders]struct {
154 item stackpoolItem
155 _ [(cpu.CacheLinePadSize - unsafe.Sizeof(stackpoolItem{})%cpu.CacheLinePadSize) % cpu.CacheLinePadSize]byte
156 }
157
158 type stackpoolItem struct {
159 _ sys.NotInHeap
160 mu mutex
161 span mSpanList
162 }
163
164 // Global pool of large stack spans.
165 var stackLarge struct {
166 lock mutex
167 free [heapAddrBits - gc.PageShift]mSpanList // free lists by log_2(s.npages)
168 }
169
170 func stackinit() {
171 if _StackCacheSize&pageMask != 0 {
172 throw("cache size must be a multiple of page size")
173 }
174 for i := range stackpool {
175 stackpool[i].item.span.init()
176 lockInit(&stackpool[i].item.mu, lockRankStackpool)
177 }
178 for i := range stackLarge.free {
179 stackLarge.free[i].init()
180 lockInit(&stackLarge.lock, lockRankStackLarge)
181 }
182 }
183
184 // stacklog2 returns ⌊log_2(n)⌋.
185 func stacklog2(n uintptr) int {
186 if n == 0 {
187 return 0
188 }
189 return bits.Len64(uint64(n))
190 }
191
192 // Allocates a stack from the free pool. Must be called with
193 // stackpool[order].item.mu held.
194 func stackpoolalloc(order uint8) gclinkptr {
195 list := &stackpool[order].item.span
196 s := list.first
197 lockWithRankMayAcquire(&mheap_.lock, lockRankMheap)
198 if s == nil {
199 // no free stacks. Allocate another span worth.
200 s = mheap_.allocManual(_StackCacheSize>>gc.PageShift, spanAllocStack)
201 if s == nil {
202 throw("out of memory")
203 }
204 if s.allocCount != 0 {
205 throw("bad allocCount")
206 }
207 if s.manualFreeList.ptr() != nil {
208 throw("bad manualFreeList")
209 }
210 osStackAlloc(s)
211 s.elemsize = fixedStack << order
212 for i := uintptr(0); i < _StackCacheSize; i += s.elemsize {
213 x := gclinkptr(s.base() + i)
214 if valgrindenabled {
215 // The address of x.ptr() becomes the base of stacks. We need to
216 // mark it allocated here and in stackfree and stackpoolfree, and free'd in
217 // stackalloc in order to avoid overlapping allocations and
218 // uninitialized memory errors in valgrind.
219 valgrindMalloc(unsafe.Pointer(x.ptr()), unsafe.Sizeof(x.ptr()))
220 }
221 x.ptr().next = s.manualFreeList
222 s.manualFreeList = x
223 }
224 list.insert(s)
225 }
226 x := s.manualFreeList
227 if x.ptr() == nil {
228 throw("span has no free stacks")
229 }
230 s.manualFreeList = x.ptr().next
231 s.allocCount++
232 if s.manualFreeList.ptr() == nil {
233 // all stacks in s are allocated.
234 list.remove(s)
235 }
236 return x
237 }
238
239 // Adds stack x to the free pool. Must be called with stackpool[order].item.mu held.
240 func stackpoolfree(x gclinkptr, order uint8) {
241 s := spanOfUnchecked(uintptr(x))
242 if s.state.get() != mSpanManual {
243 throw("freeing stack not in a stack span")
244 }
245 if s.manualFreeList.ptr() == nil {
246 // s will now have a free stack
247 stackpool[order].item.span.insert(s)
248 }
249 x.ptr().next = s.manualFreeList
250 s.manualFreeList = x
251 s.allocCount--
252 if gcphase == _GCoff && s.allocCount == 0 {
253 // Span is completely free. Return it to the heap
254 // immediately if we're sweeping.
255 //
256 // If GC is active, we delay the free until the end of
257 // GC to avoid the following type of situation:
258 //
259 // 1) GC starts, scans a SudoG but does not yet mark the SudoG.elem pointer
260 // 2) The stack that pointer points to is copied
261 // 3) The old stack is freed
262 // 4) The containing span is marked free
263 // 5) GC attempts to mark the SudoG.elem pointer. The
264 // marking fails because the pointer looks like a
265 // pointer into a free span.
266 //
267 // By not freeing, we prevent step #4 until GC is done.
268 stackpool[order].item.span.remove(s)
269 s.manualFreeList = 0
270 osStackFree(s)
271 mheap_.freeManual(s, spanAllocStack)
272 }
273 }
274
275 // stackcacherefill/stackcacherelease implement a global pool of stack segments.
276 // The pool is required to prevent unlimited growth of per-thread caches.
277 //
278 //go:systemstack
279 func stackcacherefill(c *mcache, order uint8) {
280 if stackDebug >= 1 {
281 print("stackcacherefill order=", order, "\n")
282 }
283
284 // Grab some stacks from the global cache.
285 // Grab half of the allowed capacity (to prevent thrashing).
286 var list gclinkptr
287 var size uintptr
288 lock(&stackpool[order].item.mu)
289 for size < _StackCacheSize/2 {
290 x := stackpoolalloc(order)
291 x.ptr().next = list
292 list = x
293 size += fixedStack << order
294 }
295 unlock(&stackpool[order].item.mu)
296 c.stackcache[order].list = list
297 c.stackcache[order].size = size
298 }
299
300 //go:systemstack
301 func stackcacherelease(c *mcache, order uint8) {
302 if stackDebug >= 1 {
303 print("stackcacherelease order=", order, "\n")
304 }
305 x := c.stackcache[order].list
306 size := c.stackcache[order].size
307 lock(&stackpool[order].item.mu)
308 for size > _StackCacheSize/2 {
309 y := x.ptr().next
310 stackpoolfree(x, order)
311 x = y
312 size -= fixedStack << order
313 }
314 unlock(&stackpool[order].item.mu)
315 c.stackcache[order].list = x
316 c.stackcache[order].size = size
317 }
318
319 //go:systemstack
320 func stackcache_clear(c *mcache) {
321 if stackDebug >= 1 {
322 print("stackcache clear\n")
323 }
324 for order := uint8(0); order < _NumStackOrders; order++ {
325 lock(&stackpool[order].item.mu)
326 x := c.stackcache[order].list
327 for x.ptr() != nil {
328 y := x.ptr().next
329 stackpoolfree(x, order)
330 x = y
331 }
332 c.stackcache[order].list = 0
333 c.stackcache[order].size = 0
334 unlock(&stackpool[order].item.mu)
335 }
336 }
337
338 // stackalloc allocates an n byte stack.
339 //
340 // stackalloc must run on the system stack because it uses per-P
341 // resources and must not split the stack.
342 //
343 //go:systemstack
344 func stackalloc(n uint32) stack {
345 // Stackalloc must be called on scheduler stack, so that we
346 // never try to grow the stack during the code that stackalloc runs.
347 // Doing so would cause a deadlock (issue 1547).
348 thisg := getg()
349 if thisg != thisg.m.g0 {
350 throw("stackalloc not on scheduler stack")
351 }
352 if n&(n-1) != 0 {
353 throw("stack size not a power of 2")
354 }
355 if stackDebug >= 1 {
356 print("stackalloc ", n, "\n")
357 }
358
359 if debug.efence != 0 || stackFromSystem != 0 {
360 n = uint32(alignUp(uintptr(n), physPageSize))
361 v := sysAlloc(uintptr(n), &memstats.stacks_sys, "goroutine stack (system)")
362 if v == nil {
363 throw("out of memory (stackalloc)")
364 }
365 return stack{uintptr(v), uintptr(v) + uintptr(n)}
366 }
367
368 // Small stacks are allocated with a fixed-size free-list allocator.
369 // If we need a stack of a bigger size, we fall back on allocating
370 // a dedicated span.
371 var v unsafe.Pointer
372 if n < fixedStack<<_NumStackOrders && n < _StackCacheSize {
373 order := uint8(0)
374 n2 := n
375 for n2 > fixedStack {
376 order++
377 n2 >>= 1
378 }
379 var x gclinkptr
380 if stackNoCache != 0 || thisg.m.p == 0 || thisg.m.preemptoff != "" {
381 // thisg.m.p == 0 can happen in the guts of exitsyscall
382 // or procresize. Just get a stack from the global pool.
383 // Also don't touch stackcache during gc
384 // as it's flushed concurrently.
385 lock(&stackpool[order].item.mu)
386 x = stackpoolalloc(order)
387 unlock(&stackpool[order].item.mu)
388 } else {
389 c := thisg.m.p.ptr().mcache
390 x = c.stackcache[order].list
391 if x.ptr() == nil {
392 stackcacherefill(c, order)
393 x = c.stackcache[order].list
394 }
395 c.stackcache[order].list = x.ptr().next
396 c.stackcache[order].size -= uintptr(n)
397 }
398 if valgrindenabled {
399 // We're about to allocate the stack region starting at x.ptr().
400 // To prevent valgrind from complaining about overlapping allocations,
401 // we need to mark the (previously allocated) memory as free'd.
402 valgrindFree(unsafe.Pointer(x.ptr()))
403 }
404 v = unsafe.Pointer(x)
405 } else {
406 var s *mspan
407 npage := uintptr(n) >> gc.PageShift
408 log2npage := stacklog2(npage)
409
410 // Try to get a stack from the large stack cache.
411 lock(&stackLarge.lock)
412 if !stackLarge.free[log2npage].isEmpty() {
413 s = stackLarge.free[log2npage].first
414 stackLarge.free[log2npage].remove(s)
415 }
416 unlock(&stackLarge.lock)
417
418 lockWithRankMayAcquire(&mheap_.lock, lockRankMheap)
419
420 if s == nil {
421 // Allocate a new stack from the heap.
422 s = mheap_.allocManual(npage, spanAllocStack)
423 if s == nil {
424 throw("out of memory")
425 }
426 osStackAlloc(s)
427 s.elemsize = uintptr(n)
428 }
429 v = unsafe.Pointer(s.base())
430 }
431
432 if traceAllocFreeEnabled() {
433 trace := traceAcquire()
434 if trace.ok() {
435 trace.GoroutineStackAlloc(uintptr(v), uintptr(n))
436 traceRelease(trace)
437 }
438 }
439 if raceenabled {
440 racemalloc(v, uintptr(n))
441 }
442 if msanenabled {
443 msanmalloc(v, uintptr(n))
444 }
445 if asanenabled {
446 asanunpoison(v, uintptr(n))
447 }
448 if valgrindenabled {
449 valgrindMalloc(v, uintptr(n))
450 }
451 if stackDebug >= 1 {
452 print(" allocated ", v, "\n")
453 }
454 return stack{uintptr(v), uintptr(v) + uintptr(n)}
455 }
456
457 // stackfree frees an n byte stack allocation at stk.
458 //
459 // stackfree must run on the system stack because it uses per-P
460 // resources and must not split the stack.
461 //
462 //go:systemstack
463 func stackfree(stk stack) {
464 gp := getg()
465 v := unsafe.Pointer(stk.lo)
466 n := stk.hi - stk.lo
467 if n&(n-1) != 0 {
468 throw("stack not a power of 2")
469 }
470 if stk.lo+n < stk.hi {
471 throw("bad stack size")
472 }
473 if stackDebug >= 1 {
474 println("stackfree", v, n)
475 memclrNoHeapPointers(v, n) // for testing, clobber stack data
476 }
477 if debug.efence != 0 || stackFromSystem != 0 {
478 if debug.efence != 0 || stackFaultOnFree != 0 {
479 sysFault(v, n)
480 } else {
481 sysFree(v, n, &memstats.stacks_sys)
482 }
483 return
484 }
485 if traceAllocFreeEnabled() {
486 trace := traceAcquire()
487 if trace.ok() {
488 trace.GoroutineStackFree(uintptr(v))
489 traceRelease(trace)
490 }
491 }
492 if msanenabled {
493 msanfree(v, n)
494 }
495 if asanenabled {
496 asanpoison(v, n)
497 }
498 if valgrindenabled {
499 valgrindFree(v)
500 }
501 if n < fixedStack<<_NumStackOrders && n < _StackCacheSize {
502 order := uint8(0)
503 n2 := n
504 for n2 > fixedStack {
505 order++
506 n2 >>= 1
507 }
508 x := gclinkptr(v)
509 if stackNoCache != 0 || gp.m.p == 0 || gp.m.preemptoff != "" {
510 lock(&stackpool[order].item.mu)
511 if valgrindenabled {
512 // x.ptr() is the head of the list of free stacks, and will be used
513 // when allocating a new stack, so it has to be marked allocated.
514 valgrindMalloc(unsafe.Pointer(x.ptr()), unsafe.Sizeof(x.ptr()))
515 }
516 stackpoolfree(x, order)
517 unlock(&stackpool[order].item.mu)
518 } else {
519 c := gp.m.p.ptr().mcache
520 if c.stackcache[order].size >= _StackCacheSize {
521 stackcacherelease(c, order)
522 }
523 if valgrindenabled {
524 // x.ptr() is the head of the list of free stacks, and will
525 // be used when allocating a new stack, so it has to be
526 // marked allocated.
527 valgrindMalloc(unsafe.Pointer(x.ptr()), unsafe.Sizeof(x.ptr()))
528 }
529 x.ptr().next = c.stackcache[order].list
530 c.stackcache[order].list = x
531 c.stackcache[order].size += n
532 }
533 } else {
534 s := spanOfUnchecked(uintptr(v))
535 if s.state.get() != mSpanManual {
536 println(hex(s.base()), v)
537 throw("bad span state")
538 }
539 if gcphase == _GCoff {
540 // Free the stack immediately if we're
541 // sweeping.
542 osStackFree(s)
543 mheap_.freeManual(s, spanAllocStack)
544 } else {
545 // If the GC is running, we can't return a
546 // stack span to the heap because it could be
547 // reused as a heap span, and this state
548 // change would race with GC. Add it to the
549 // large stack cache instead.
550 log2npage := stacklog2(s.npages)
551 lock(&stackLarge.lock)
552 stackLarge.free[log2npage].insert(s)
553 unlock(&stackLarge.lock)
554 }
555 }
556 }
557
558 var maxstacksize uintptr = 1 << 20 // enough until runtime.main sets it for real
559
560 var maxstackceiling = maxstacksize
561
562 var ptrnames = []string{
563 0: "scalar",
564 1: "ptr",
565 }
566
567 // Stack frame layout
568 //
569 // (x86)
570 // +------------------+
571 // | args from caller |
572 // +------------------+ <- frame->argp
573 // | return address |
574 // +------------------+
575 // | caller's BP (*) | (*) if framepointer_enabled && varp > sp
576 // +------------------+ <- frame->varp
577 // | locals |
578 // +------------------+
579 // | args to callee |
580 // +------------------+ <- frame->sp
581 //
582 // (arm)
583 // +------------------+
584 // | args from caller |
585 // +------------------+ <- frame->argp
586 // | caller's retaddr |
587 // +------------------+
588 // | caller's FP (*) | (*) on ARM64, if framepointer_enabled && varp > sp
589 // +------------------+ <- frame->varp
590 // | locals |
591 // +------------------+
592 // | args to callee |
593 // +------------------+
594 // | return address |
595 // +------------------+ <- frame->sp
596 //
597 // varp > sp means that the function has a frame;
598 // varp == sp means frameless function.
599
600 type adjustinfo struct {
601 old stack
602 delta uintptr // ptr distance from old to new stack (newbase - oldbase)
603
604 // sghi is the highest sudog.elem on the stack.
605 sghi uintptr
606 }
607
608 // adjustpointer checks whether *vpp is in the old stack described by adjinfo.
609 // If so, it rewrites *vpp to point into the new stack.
610 func adjustpointer(adjinfo *adjustinfo, vpp unsafe.Pointer) {
611 pp := (*uintptr)(vpp)
612 p := *pp
613 if stackDebug >= 4 {
614 print(" ", pp, ":", hex(p), "\n")
615 }
616 if valgrindenabled {
617 // p is a pointer on a stack, it is inherently initialized, as
618 // everything on the stack is, but valgrind for _some unknown reason_
619 // sometimes thinks it's uninitialized, and flags operations on p below
620 // as uninitialized. We just initialize it if valgrind thinks its
621 // uninitialized.
622 //
623 // See go.dev/issues/73801.
624 valgrindMakeMemDefined(unsafe.Pointer(&p), unsafe.Sizeof(&p))
625 }
626 if adjinfo.old.lo <= p && p < adjinfo.old.hi {
627 *pp = p + adjinfo.delta
628 if stackDebug >= 3 {
629 print(" adjust ptr ", pp, ":", hex(p), " -> ", hex(*pp), "\n")
630 }
631 }
632 }
633
634 // Information from the compiler about the layout of stack frames.
635 // Note: this type must agree with reflect.bitVector.
636 type bitvector struct {
637 n int32 // # of bits
638 bytedata *uint8
639 }
640
641 // ptrbit returns the i'th bit in bv.
642 // ptrbit is less efficient than iterating directly over bitvector bits,
643 // and should only be used in non-performance-critical code.
644 // See adjustpointers for an example of a high-efficiency walk of a bitvector.
645 func (bv *bitvector) ptrbit(i uintptr) uint8 {
646 b := *(addb(bv.bytedata, i/8))
647 return (b >> (i % 8)) & 1
648 }
649
650 // bv describes the memory starting at address scanp.
651 // Adjust any pointers contained therein.
652 func adjustpointers(scanp unsafe.Pointer, bv *bitvector, adjinfo *adjustinfo, f funcInfo) {
653 minp := adjinfo.old.lo
654 maxp := adjinfo.old.hi
655 delta := adjinfo.delta
656 num := uintptr(bv.n)
657 // If this frame might contain channel receive slots, use CAS
658 // to adjust pointers. If the slot hasn't been received into
659 // yet, it may contain stack pointers and a concurrent send
660 // could race with adjusting those pointers. (The sent value
661 // itself can never contain stack pointers.)
662 useCAS := uintptr(scanp) < adjinfo.sghi
663 for i := uintptr(0); i < num; i += 8 {
664 if stackDebug >= 4 {
665 for j := uintptr(0); j < 8; j++ {
666 print(" ", add(scanp, (i+j)*goarch.PtrSize), ":", ptrnames[bv.ptrbit(i+j)], ":", hex(*(*uintptr)(add(scanp, (i+j)*goarch.PtrSize))), " # ", i, " ", *addb(bv.bytedata, i/8), "\n")
667 }
668 }
669 b := *(addb(bv.bytedata, i/8))
670 for b != 0 {
671 j := uintptr(sys.TrailingZeros8(b))
672 b &= b - 1
673 pp := (*uintptr)(add(scanp, (i+j)*goarch.PtrSize))
674 retry:
675 p := *pp
676 if f.valid() && 0 < p && p < minLegalPointer && debug.invalidptr != 0 {
677 // Looks like a junk value in a pointer slot.
678 // Live analysis wrong?
679 getg().m.traceback = 2
680 print("runtime: bad pointer in frame ", funcname(f), " at ", pp, ": ", hex(p), "\n")
681 throw("invalid pointer found on stack")
682 }
683 if minp <= p && p < maxp {
684 if stackDebug >= 3 {
685 print("adjust ptr ", hex(p), " ", funcname(f), "\n")
686 }
687 if useCAS {
688 ppu := (*unsafe.Pointer)(unsafe.Pointer(pp))
689 if !atomic.Casp1(ppu, unsafe.Pointer(p), unsafe.Pointer(p+delta)) {
690 goto retry
691 }
692 } else {
693 *pp = p + delta
694 }
695 }
696 }
697 }
698 }
699
700 // Note: the argument/return area is adjusted by the callee.
701 func adjustframe(frame *stkframe, adjinfo *adjustinfo) {
702 // Adjust saved frame pointer if there is one.
703 if (goarch.ArchFamily == goarch.AMD64 || goarch.ArchFamily == goarch.ARM64) && frame.argp-frame.varp == 2*goarch.PtrSize {
704 if stackDebug >= 3 {
705 print(" saved bp\n")
706 }
707 if debugCheckBP {
708 // Frame pointers should always point to the next higher frame on
709 // the Go stack (or be nil, for the top frame on the stack).
710 bp := *(*uintptr)(unsafe.Pointer(frame.varp))
711 if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) {
712 println("runtime: found invalid frame pointer")
713 print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n")
714 throw("bad frame pointer")
715 }
716 }
717 // On AMD64, this is the caller's frame pointer saved in the current
718 // frame.
719 // On ARM64, this is the frame pointer of the caller's caller saved
720 // by the caller in its frame (one word below its SP).
721 adjustpointer(adjinfo, unsafe.Pointer(frame.varp))
722 }
723 if goarch.ArchFamily == goarch.ARM64 && isInjectedCall(frame.fn.funcID) {
724 // If this is an injected call on arm64, then we need to adjust
725 // the frame pointer saved by the original function into which
726 // the call was injected. Normally this would be handled when
727 // adjusting the callee's frame or in adjustctxt. But when a
728 // call is injected, the frame is placed 16 bytes below the
729 // original stack pointer to make room to save the link
730 // register, and the frame pointer saved by the original
731 // function isn't inside any call frame. We can adjust that
732 // saved frame pointer here by looking just above frame.fp.
733 //
734 // ^ original call ^
735 // | frame above... |
736 // +-------------------+ <- stack pointer at the time of injection
737 // : FP saved by :
738 // : original func :
739 // :···················: <- frame pointer register from original function
740 // : LR saved during :
741 // : injection :
742 // +-------------------+ <- frame.fp (injection decrements SP by 16 bytes)
743 // | FP saved |
744 // | during injection |
745 // +-------------------+
746 // | injected call |
747 // V frame below... V
748 adjustpointer(adjinfo, unsafe.Pointer(frame.fp+goarch.PtrSize))
749 }
750
751 if frame.continpc == 0 {
752 // Frame is dead. The program might still see the frame pointer
753 // saved in the frame, adjusted above, but we don't need to
754 // adjust the rest of the frame.
755 return
756 }
757 f := frame.fn
758 if stackDebug >= 2 {
759 print(" adjusting ", funcname(f), " frame=[", hex(frame.sp), ",", hex(frame.fp), "] pc=", hex(frame.pc), " continpc=", hex(frame.continpc), "\n")
760 }
761
762 locals, args, objs := frame.getStackMap(true)
763
764 // Adjust local variables if stack frame has been allocated.
765 if locals.n > 0 {
766 size := uintptr(locals.n) * goarch.PtrSize
767 adjustpointers(unsafe.Pointer(frame.varp-size), &locals, adjinfo, f)
768 }
769
770 // Adjust arguments.
771 if args.n > 0 {
772 if stackDebug >= 3 {
773 print(" args\n")
774 }
775 adjustpointers(unsafe.Pointer(frame.argp), &args, adjinfo, funcInfo{})
776 }
777
778 // Adjust pointers in all stack objects (whether they are live or not).
779 // See comments in mgcmark.go:scanframeworker.
780 if frame.varp != 0 {
781 for i := range objs {
782 obj := &objs[i]
783 off := obj.off
784 base := frame.varp // locals base pointer
785 if off >= 0 {
786 base = frame.argp // arguments and return values base pointer
787 }
788 p := base + uintptr(off)
789 if p < frame.sp {
790 // Object hasn't been allocated in the frame yet.
791 // (Happens when the stack bounds check fails and
792 // we call into morestack.)
793 continue
794 }
795 ptrBytes, gcData := obj.gcdata()
796 for i := uintptr(0); i < ptrBytes; i += goarch.PtrSize {
797 if *addb(gcData, i/(8*goarch.PtrSize))>>(i/goarch.PtrSize&7)&1 != 0 {
798 adjustpointer(adjinfo, unsafe.Pointer(p+i))
799 }
800 }
801 }
802 }
803 }
804
805 func adjustctxt(gp *g, adjinfo *adjustinfo) {
806 adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.ctxt))
807 if !framepointer_enabled {
808 return
809 }
810 if debugCheckBP {
811 bp := gp.sched.bp
812 if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) {
813 println("runtime: found invalid top frame pointer")
814 print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n")
815 throw("bad top frame pointer")
816 }
817 }
818 oldfp := gp.sched.bp
819 adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.bp))
820 if GOARCH == "arm64" {
821 // On ARM64, the frame pointer is saved one word *below* the SP,
822 // which is not copied or adjusted in any frame. Do it explicitly
823 // here.
824 if oldfp == gp.sched.sp-goarch.PtrSize {
825 memmove(unsafe.Pointer(gp.sched.bp), unsafe.Pointer(oldfp), goarch.PtrSize)
826 adjustpointer(adjinfo, unsafe.Pointer(gp.sched.bp))
827 }
828 }
829 }
830
831 func adjustdefers(gp *g, adjinfo *adjustinfo) {
832 // Adjust pointers in the Defer structs.
833 // We need to do this first because we need to adjust the
834 // defer.link fields so we always work on the new stack.
835 adjustpointer(adjinfo, unsafe.Pointer(&gp._defer))
836 for d := gp._defer; d != nil; d = d.link {
837 adjustpointer(adjinfo, unsafe.Pointer(&d.fn))
838 adjustpointer(adjinfo, unsafe.Pointer(&d.sp))
839 adjustpointer(adjinfo, unsafe.Pointer(&d.link))
840 }
841 }
842
843 func adjustpanics(gp *g, adjinfo *adjustinfo) {
844 // Panics are on stack and already adjusted.
845 // Update pointer to head of list in G.
846 adjustpointer(adjinfo, unsafe.Pointer(&gp._panic))
847 }
848
849 func adjustsudogs(gp *g, adjinfo *adjustinfo) {
850 // the data elements pointed to by a SudoG structure
851 // might be in the stack.
852 for s := gp.waiting; s != nil; s = s.waitlink {
853 adjustpointer(adjinfo, unsafe.Pointer(&s.elem.vu))
854 adjustpointer(adjinfo, unsafe.Pointer(&s.elem.vp))
855 }
856 }
857
858 func fillstack(stk stack, b byte) {
859 for p := stk.lo; p < stk.hi; p++ {
860 *(*byte)(unsafe.Pointer(p)) = b
861 }
862 }
863
864 func findsghi(gp *g, stk stack) uintptr {
865 var sghi uintptr
866 for sg := gp.waiting; sg != nil; sg = sg.waitlink {
867 p := sg.elem.uintptr() + uintptr(sg.c.get().elemsize)
868 if stk.lo <= p && p < stk.hi && p > sghi {
869 sghi = p
870 }
871 }
872 return sghi
873 }
874
875 // syncadjustsudogs adjusts gp's sudogs and copies the part of gp's
876 // stack they refer to while synchronizing with concurrent channel
877 // operations. It returns the number of bytes of stack copied.
878 func syncadjustsudogs(gp *g, used uintptr, adjinfo *adjustinfo) uintptr {
879 if gp.waiting == nil {
880 return 0
881 }
882
883 // Lock channels to prevent concurrent send/receive.
884 var lastc *hchan
885 for sg := gp.waiting; sg != nil; sg = sg.waitlink {
886 if sg.c.get() != lastc {
887 // There is a ranking cycle here between gscan bit and
888 // hchan locks. Normally, we only allow acquiring hchan
889 // locks and then getting a gscan bit. In this case, we
890 // already have the gscan bit. We allow acquiring hchan
891 // locks here as a special case, since a deadlock can't
892 // happen because the G involved must already be
893 // suspended. So, we get a special hchan lock rank here
894 // that is lower than gscan, but doesn't allow acquiring
895 // any other locks other than hchan.
896 lockWithRank(&sg.c.get().lock, lockRankHchanLeaf)
897 }
898 lastc = sg.c.get()
899 }
900
901 // Adjust sudogs.
902 adjustsudogs(gp, adjinfo)
903
904 // Copy the part of the stack the sudogs point in to
905 // while holding the lock to prevent races on
906 // send/receive slots.
907 var sgsize uintptr
908 if adjinfo.sghi != 0 {
909 oldBot := adjinfo.old.hi - used
910 newBot := oldBot + adjinfo.delta
911 sgsize = adjinfo.sghi - oldBot
912 memmove(unsafe.Pointer(newBot), unsafe.Pointer(oldBot), sgsize)
913 }
914
915 // Unlock channels.
916 lastc = nil
917 for sg := gp.waiting; sg != nil; sg = sg.waitlink {
918 if sg.c.get() != lastc {
919 unlock(&sg.c.get().lock)
920 }
921 lastc = sg.c.get()
922 }
923
924 return sgsize
925 }
926
927 // Copies gp's stack to a new stack of a different size.
928 // Caller must have changed gp status to Gcopystack.
929 func copystack(gp *g, newsize uintptr) {
930 if gp.syscallsp != 0 {
931 throw("stack growth not allowed in system call")
932 }
933 old := gp.stack
934 if old.lo == 0 {
935 throw("nil stackbase")
936 }
937 used := old.hi - gp.sched.sp
938 // Add just the difference to gcController.addScannableStack.
939 // g0 stacks never move, so this will never account for them.
940 // It's also fine if we have no P, addScannableStack can deal with
941 // that case.
942 gcController.addScannableStack(getg().m.p.ptr(), int64(newsize)-int64(old.hi-old.lo))
943
944 // allocate new stack
945 new := stackalloc(uint32(newsize))
946 if stackPoisonCopy != 0 {
947 fillstack(new, 0xfd)
948 }
949 if stackDebug >= 1 {
950 print("copystack gp=", gp, " [", hex(old.lo), " ", hex(old.hi-used), " ", hex(old.hi), "]", " -> [", hex(new.lo), " ", hex(new.hi-used), " ", hex(new.hi), "]/", newsize, "\n")
951 }
952
953 // Compute adjustment.
954 var adjinfo adjustinfo
955 adjinfo.old = old
956 adjinfo.delta = new.hi - old.hi
957
958 // Adjust sudogs, synchronizing with channel ops if necessary.
959 ncopy := used
960 if !gp.activeStackChans {
961 if newsize < old.hi-old.lo && gp.parkingOnChan.Load() {
962 // It's not safe for someone to shrink this stack while we're actively
963 // parking on a channel, but it is safe to grow since we do that
964 // ourselves and explicitly don't want to synchronize with channels
965 // since we could self-deadlock.
966 throw("racy sudog adjustment due to parking on channel")
967 }
968 adjustsudogs(gp, &adjinfo)
969 } else {
970 // sudogs may be pointing in to the stack and gp has
971 // released channel locks, so other goroutines could
972 // be writing to gp's stack. Find the highest such
973 // pointer so we can handle everything there and below
974 // carefully. (This shouldn't be far from the bottom
975 // of the stack, so there's little cost in handling
976 // everything below it carefully.)
977 adjinfo.sghi = findsghi(gp, old)
978
979 // Synchronize with channel ops and copy the part of
980 // the stack they may interact with.
981 ncopy -= syncadjustsudogs(gp, used, &adjinfo)
982 }
983
984 // Copy the stack (or the rest of it) to the new location
985 memmove(unsafe.Pointer(new.hi-ncopy), unsafe.Pointer(old.hi-ncopy), ncopy)
986
987 // Adjust remaining structures that have pointers into stacks.
988 // We have to do most of these before we traceback the new
989 // stack because gentraceback uses them.
990 adjustctxt(gp, &adjinfo)
991 adjustdefers(gp, &adjinfo)
992 adjustpanics(gp, &adjinfo)
993 if adjinfo.sghi != 0 {
994 adjinfo.sghi += adjinfo.delta
995 }
996
997 // Swap out old stack for new one
998 gp.stack = new
999 gp.stackguard0 = new.lo + stackGuard // NOTE: might clobber a preempt request
1000 gp.sched.sp = new.hi - used
1001 gp.stktopsp += adjinfo.delta
1002
1003 // Adjust pointers in the new stack.
1004 var u unwinder
1005 for u.init(gp, 0); u.valid(); u.next() {
1006 adjustframe(&u.frame, &adjinfo)
1007 }
1008
1009 if valgrindenabled {
1010 if gp.valgrindStackID == 0 {
1011 gp.valgrindStackID = valgrindRegisterStack(unsafe.Pointer(new.lo), unsafe.Pointer(new.hi))
1012 } else {
1013 valgrindChangeStack(gp.valgrindStackID, unsafe.Pointer(new.lo), unsafe.Pointer(new.hi))
1014 }
1015 }
1016
1017 // free old stack
1018 if goexperiment.RuntimeSecret && gp.secret > 0 {
1019 // Some portion of the old stack has secret stuff on it.
1020 // We don't really know where we entered secret mode,
1021 // so just clear the whole thing.
1022 // TODO(dmo): traceback until we hit secret.Do? clearing
1023 // is fast and optimized, might not be worth it.
1024 memclrNoHeapPointers(unsafe.Pointer(old.lo), old.hi-old.lo)
1025 // The memmove call above might put secrets from the stack into registers.
1026 secretEraseRegisters()
1027 }
1028 if stackPoisonCopy != 0 {
1029 fillstack(old, 0xfc)
1030 }
1031 stackfree(old)
1032 }
1033
1034 // round x up to a power of 2.
1035 func round2(x int32) int32 {
1036 s := uint(0)
1037 for 1<<s < x {
1038 s++
1039 }
1040 return 1 << s
1041 }
1042
1043 // Called from runtime·morestack when more stack is needed.
1044 // Allocate larger stack and relocate to new stack.
1045 // Stack growth is multiplicative, for constant amortized cost.
1046 //
1047 // g->atomicstatus will be Grunning or Gscanrunning upon entry.
1048 // If the scheduler is trying to stop this g, then it will set preemptStop.
1049 //
1050 // This must be nowritebarrierrec because it can be called as part of
1051 // stack growth from other nowritebarrierrec functions, but the
1052 // compiler doesn't check this.
1053 //
1054 //go:nowritebarrierrec
1055 func newstack() {
1056 thisg := getg()
1057 // TODO: double check all gp. shouldn't be getg().
1058 if thisg.m.morebuf.g.ptr().stackguard0 == stackFork {
1059 throw("stack growth after fork")
1060 }
1061 if thisg.m.morebuf.g.ptr() != thisg.m.curg {
1062 print("runtime: newstack called from g=", hex(thisg.m.morebuf.g), "\n"+"\tm=", thisg.m, " m->curg=", thisg.m.curg, " m->g0=", thisg.m.g0, " m->gsignal=", thisg.m.gsignal, "\n")
1063 morebuf := thisg.m.morebuf
1064 traceback(morebuf.pc, morebuf.sp, morebuf.lr, morebuf.g.ptr())
1065 throw("runtime: wrong goroutine in newstack")
1066 }
1067
1068 gp := thisg.m.curg
1069 if goexperiment.RuntimeSecret && gp.secret > 0 {
1070 // If we're entering here from a secret context, clear
1071 // all the registers. This is important because we
1072 // might context switch to a different goroutine which
1073 // is not in secret mode, and it will not be careful
1074 // about clearing its registers.
1075 secretEraseRegisters()
1076 }
1077
1078 if thisg.m.curg.throwsplit {
1079 // Update syscallsp, syscallpc in case traceback uses them.
1080 morebuf := thisg.m.morebuf
1081 gp.syscallsp = morebuf.sp
1082 gp.syscallpc = morebuf.pc
1083 pcname, pcoff := "(unknown)", uintptr(0)
1084 f := findfunc(gp.sched.pc)
1085 if f.valid() {
1086 pcname = funcname(f)
1087 pcoff = gp.sched.pc - f.entry()
1088 }
1089 print("runtime: newstack at ", pcname, "+", hex(pcoff),
1090 " sp=", hex(gp.sched.sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n",
1091 "\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n",
1092 "\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n")
1093
1094 thisg.m.traceback = 2 // Include runtime frames
1095 traceback(morebuf.pc, morebuf.sp, morebuf.lr, gp)
1096 throw("runtime: stack split at bad time")
1097 }
1098
1099 morebuf := thisg.m.morebuf
1100 thisg.m.morebuf.pc = 0
1101 thisg.m.morebuf.lr = 0
1102 thisg.m.morebuf.sp = 0
1103 thisg.m.morebuf.g = 0
1104
1105 // NOTE: stackguard0 may change underfoot, if another thread
1106 // is about to try to preempt gp. Read it just once and use that same
1107 // value now and below.
1108 stackguard0 := atomic.Loaduintptr(&gp.stackguard0)
1109
1110 // Be conservative about where we preempt.
1111 // We are interested in preempting user Go code, not runtime code.
1112 // If we're holding locks, mallocing, or preemption is disabled, don't
1113 // preempt.
1114 // This check is very early in newstack so that even the status change
1115 // from Grunning to Gwaiting and back doesn't happen in this case.
1116 // That status change by itself can be viewed as a small preemption,
1117 // because the GC might change Gwaiting to Gscanwaiting, and then
1118 // this goroutine has to wait for the GC to finish before continuing.
1119 // If the GC is in some way dependent on this goroutine (for example,
1120 // it needs a lock held by the goroutine), that small preemption turns
1121 // into a real deadlock.
1122 preempt := stackguard0 == stackPreempt
1123 if preempt {
1124 if !canPreemptM(thisg.m) {
1125 // Let the goroutine keep running for now.
1126 // gp->preempt is set, so it will be preempted next time.
1127 gp.stackguard0 = gp.stack.lo + stackGuard
1128 gogo(&gp.sched) // never return
1129 }
1130 }
1131
1132 if gp.stack.lo == 0 {
1133 throw("missing stack in newstack")
1134 }
1135 sp := gp.sched.sp
1136 if goarch.ArchFamily == goarch.AMD64 || goarch.ArchFamily == goarch.I386 || goarch.ArchFamily == goarch.WASM {
1137 // The call to morestack cost a word.
1138 sp -= goarch.PtrSize
1139 }
1140 if stackDebug >= 1 || sp < gp.stack.lo {
1141 print("runtime: newstack sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n",
1142 "\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n",
1143 "\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n")
1144 }
1145 if sp < gp.stack.lo {
1146 print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->status=", hex(readgstatus(gp)), "\n ")
1147 print("runtime: split stack overflow: ", hex(sp), " < ", hex(gp.stack.lo), "\n")
1148 throw("runtime: split stack overflow")
1149 }
1150
1151 if preempt {
1152 if gp == thisg.m.g0 {
1153 throw("runtime: preempt g0")
1154 }
1155 if thisg.m.p == 0 && thisg.m.locks == 0 {
1156 throw("runtime: g is running but p is not")
1157 }
1158
1159 if gp.preemptShrink {
1160 // We're at a synchronous safe point now, so
1161 // do the pending stack shrink.
1162 gp.preemptShrink = false
1163 shrinkstack(gp)
1164 }
1165
1166 // Set a flag indicated that we've been synchronously preempted.
1167 gp.syncSafePoint = true
1168
1169 if gp.preemptStop {
1170 preemptPark(gp) // never returns
1171 }
1172
1173 // Act like goroutine called runtime.Gosched.
1174 gopreempt_m(gp) // never return
1175 }
1176
1177 // Allocate a bigger segment and move the stack.
1178 oldsize := gp.stack.hi - gp.stack.lo
1179 newsize := oldsize * 2
1180
1181 // Make sure we grow at least as much as needed to fit the new frame.
1182 // (This is just an optimization - the caller of morestack will
1183 // recheck the bounds on return.)
1184 if f := findfunc(gp.sched.pc); f.valid() {
1185 max := uintptr(funcMaxSPDelta(f))
1186 needed := max + stackGuard
1187 used := gp.stack.hi - gp.sched.sp
1188 for newsize-used < needed {
1189 newsize *= 2
1190 }
1191 }
1192
1193 if stackguard0 == stackForceMove {
1194 // Forced stack movement used for debugging.
1195 // Don't double the stack (or we may quickly run out
1196 // if this is done repeatedly).
1197 newsize = oldsize
1198 }
1199
1200 if newsize > maxstacksize || newsize > maxstackceiling {
1201 if maxstacksize < maxstackceiling {
1202 print("runtime: goroutine stack exceeds ", maxstacksize, "-byte limit\n")
1203 } else {
1204 print("runtime: goroutine stack exceeds ", maxstackceiling, "-byte limit\n")
1205 }
1206 print("runtime: sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n")
1207 throw("stack overflow")
1208 }
1209
1210 // The goroutine must be executing in order to call newstack,
1211 // so it must be Grunning (or Gscanrunning).
1212 casgstatus(gp, _Grunning, _Gcopystack)
1213
1214 // The concurrent GC will not scan the stack while we are doing the copy since
1215 // the gp is in a Gcopystack status.
1216 copystack(gp, newsize)
1217 if stackDebug >= 1 {
1218 print("stack grow done\n")
1219 }
1220 casgstatus(gp, _Gcopystack, _Grunning)
1221 gogo(&gp.sched)
1222 }
1223
1224 //go:nosplit
1225 func nilfunc() {
1226 *(*uint8)(nil) = 0
1227 }
1228
1229 // adjust Gobuf as if it executed a call to fn
1230 // and then stopped before the first instruction in fn.
1231 func gostartcallfn(gobuf *gobuf, fv *funcval) {
1232 var fn unsafe.Pointer
1233 if fv != nil {
1234 fn = unsafe.Pointer(fv.fn)
1235 } else {
1236 fn = unsafe.Pointer(abi.FuncPCABIInternal(nilfunc))
1237 }
1238 gostartcall(gobuf, fn, unsafe.Pointer(fv))
1239 }
1240
1241 // isShrinkStackSafe returns whether it's safe to attempt to shrink
1242 // gp's stack. Shrinking the stack is only safe when we have precise
1243 // pointer maps for all frames on the stack. The caller must hold the
1244 // _Gscan bit for gp or must be running gp itself.
1245 func isShrinkStackSafe(gp *g) bool {
1246 // We can't copy the stack if we're in a syscall.
1247 // The syscall might have pointers into the stack and
1248 // often we don't have precise pointer maps for the innermost
1249 // frames.
1250 if gp.syscallsp != 0 {
1251 return false
1252 }
1253 // We also can't copy the stack if we're at an asynchronous
1254 // safe-point because we don't have precise pointer maps for
1255 // all frames.
1256 if gp.asyncSafePoint {
1257 return false
1258 }
1259 // We also can't *shrink* the stack in the window between the
1260 // goroutine calling gopark to park on a channel and
1261 // gp.activeStackChans being set.
1262 if gp.parkingOnChan.Load() {
1263 return false
1264 }
1265 // We also can't copy the stack while a gp is in _Gwaiting solely
1266 // to make itself available to suspendG.
1267 //
1268 // In these cases, the G is actually executing on the system
1269 // stack, and the execution tracer, mutex profiler, etc. may want
1270 // to take a stack trace of the G's stack.
1271 //
1272 // Note: it's safe to access gp.waitreason here.
1273 // We're only calling isShrinkStackSafe if we took ownership of the
1274 // G with the _Gscan bit. This prevents the goroutine from transitioning,
1275 // which prevents gp.waitreason from changing.
1276 if readgstatus(gp)&^_Gscan == _Gwaiting && gp.waitreason.isWaitingForSuspendG() {
1277 return false
1278 }
1279 return true
1280 }
1281
1282 // Maybe shrink the stack being used by gp.
1283 //
1284 // gp must be stopped and we must own its stack. It may be in
1285 // _Grunning, but only if this is our own user G.
1286 func shrinkstack(gp *g) {
1287 if gp.stack.lo == 0 {
1288 throw("missing stack in shrinkstack")
1289 }
1290 if s := readgstatus(gp); s&_Gscan == 0 {
1291 // We don't own the stack via _Gscan. We could still
1292 // own it if this is our own user G and we're on the
1293 // system stack.
1294 if !(gp == getg().m.curg && getg() != getg().m.curg && s == _Grunning) {
1295 // We don't own the stack.
1296 throw("bad status in shrinkstack")
1297 }
1298 }
1299 if !isShrinkStackSafe(gp) {
1300 throw("shrinkstack at bad time")
1301 }
1302 // Check for self-shrinks while in a libcall. These may have
1303 // pointers into the stack disguised as uintptrs, but these
1304 // code paths should all be nosplit.
1305 if gp == getg().m.curg && gp.m.libcallsp != 0 {
1306 throw("shrinking stack in libcall")
1307 }
1308
1309 if debug.gcshrinkstackoff > 0 {
1310 return
1311 }
1312
1313 oldsize := gp.stack.hi - gp.stack.lo
1314 newsize := oldsize / 2
1315 // Don't shrink the allocation below the minimum-sized stack
1316 // allocation.
1317 if newsize < fixedStack {
1318 return
1319 }
1320 // Compute how much of the stack is currently in use and only
1321 // shrink the stack if gp is using less than a quarter of its
1322 // current stack. The currently used stack includes everything
1323 // down to the SP plus the stack guard space that ensures
1324 // there's room for nosplit functions.
1325 avail := gp.stack.hi - gp.stack.lo
1326 if used := gp.stack.hi - gp.sched.sp + stackNosplit; used >= avail/4 {
1327 return
1328 }
1329
1330 if stackDebug > 0 {
1331 print("shrinking stack ", oldsize, "->", newsize, "\n")
1332 }
1333
1334 copystack(gp, newsize)
1335 }
1336
1337 // freeStackSpans frees unused stack spans at the end of GC.
1338 func freeStackSpans() {
1339 // Scan stack pools for empty stack spans.
1340 for order := range stackpool {
1341 lock(&stackpool[order].item.mu)
1342 list := &stackpool[order].item.span
1343 for s := list.first; s != nil; {
1344 next := s.next
1345 if s.allocCount == 0 {
1346 list.remove(s)
1347 s.manualFreeList = 0
1348 osStackFree(s)
1349 mheap_.freeManual(s, spanAllocStack)
1350 }
1351 s = next
1352 }
1353 unlock(&stackpool[order].item.mu)
1354 }
1355
1356 // Free large stack spans.
1357 lock(&stackLarge.lock)
1358 for i := range stackLarge.free {
1359 for s := stackLarge.free[i].first; s != nil; {
1360 next := s.next
1361 stackLarge.free[i].remove(s)
1362 osStackFree(s)
1363 mheap_.freeManual(s, spanAllocStack)
1364 s = next
1365 }
1366 }
1367 unlock(&stackLarge.lock)
1368 }
1369
1370 // A stackObjectRecord is generated by the compiler for each stack object in a stack frame.
1371 // This record must match the generator code in cmd/compile/internal/liveness/plive.go:emitStackObjects.
1372 type stackObjectRecord struct {
1373 // offset in frame
1374 // if negative, offset from varp
1375 // if non-negative, offset from argp
1376 off int32
1377 size int32
1378 ptrBytes int32
1379 gcdataoff uint32 // offset to gcdata from moduledata.rodata
1380 }
1381
1382 // gcdata returns the number of bytes that contain pointers, and
1383 // a ptr/nonptr bitmask covering those bytes.
1384 // Note that this bitmask might be larger than internal/abi.MaxPtrmaskBytes.
1385 func (r *stackObjectRecord) gcdata() (uintptr, *byte) {
1386 ptr := uintptr(unsafe.Pointer(r))
1387 var mod *moduledata
1388 for datap := &firstmoduledata; datap != nil; datap = datap.next {
1389 // The normal case: stackObjectRecord is in funcdata.
1390 if datap.gofunc <= ptr && ptr < datap.epclntab {
1391 mod = datap
1392 break
1393 }
1394 // A special case: methodValueCallFrameObjs.
1395 if datap.noptrbss <= ptr && ptr < datap.enoptrbss {
1396 mod = datap
1397 break
1398 }
1399 }
1400 // If you get a panic here due to a nil mod,
1401 // you may have made a copy of a stackObjectRecord.
1402 // You must use the original pointer.
1403 res := mod.rodata + uintptr(r.gcdataoff)
1404 return uintptr(r.ptrBytes), (*byte)(unsafe.Pointer(res))
1405 }
1406
1407 // This is exported as ABI0 via linkname so obj can call it.
1408 //
1409 //go:nosplit
1410 //go:linkname morestackc
1411 func morestackc() {
1412 throw("attempt to execute system stack code on user stack")
1413 }
1414
1415 // startingStackSize is the amount of stack that new goroutines start with.
1416 // It is a power of 2, and between fixedStack and maxstacksize, inclusive.
1417 // startingStackSize is updated every GC by tracking the average size of
1418 // stacks scanned during the GC.
1419 var startingStackSize uint32 = fixedStack
1420
1421 func gcComputeStartingStackSize() {
1422 if debug.adaptivestackstart == 0 {
1423 return
1424 }
1425 // For details, see the design doc at
1426 // https://docs.google.com/document/d/1YDlGIdVTPnmUiTAavlZxBI1d9pwGQgZT7IKFKlIXohQ/edit?usp=sharing
1427 // The basic algorithm is to track the average size of stacks
1428 // and start goroutines with stack equal to that average size.
1429 // Starting at the average size uses at most 2x the space that
1430 // an ideal algorithm would have used.
1431 // This is just a heuristic to avoid excessive stack growth work
1432 // early in a goroutine's lifetime. See issue 18138. Stacks that
1433 // are allocated too small can still grow, and stacks allocated
1434 // too large can still shrink.
1435 var scannedStackSize uint64
1436 var scannedStacks uint64
1437 for _, p := range allp {
1438 scannedStackSize += p.scannedStackSize
1439 scannedStacks += p.scannedStacks
1440 // Reset for next time
1441 p.scannedStackSize = 0
1442 p.scannedStacks = 0
1443 }
1444 if scannedStacks == 0 {
1445 startingStackSize = fixedStack
1446 return
1447 }
1448 avg := scannedStackSize/scannedStacks + stackGuard
1449 // Note: we add stackGuard to ensure that a goroutine that
1450 // uses the average space will not trigger a growth.
1451 if avg > uint64(maxstacksize) {
1452 avg = uint64(maxstacksize)
1453 }
1454 if avg < fixedStack {
1455 avg = fixedStack
1456 }
1457 // Note: maxstacksize fits in 30 bits, so avg also does.
1458 startingStackSize = uint32(round2(int32(avg)))
1459 }
1460
View as plain text