GeezerVisonThe World Wide Web, restored to 800 pixels of glory. |
| Home|About|Gaming cabinet|Archives|How it works|Starter sites |
GeezerVison is showing github.com — cpython/Python/jit_unwind.c at main · python/cpython Expand file tree/ Copy pathjit_unwind.cMore file actions 1062 lines (979 loc) · 41.2 KB / Copy pathjit_unwind.cFile metadata and controls1062 lines (979 loc) · 41.2 KB 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 /* * Python JIT - DWARF .eh_frame builder * * This file contains the DWARF CFI generator used to build .eh_frame * data for JIT code (perf jitdump and other unwinders). */ #include "Python.h" #include "pycore_jit_unwind.h" #include "pycore_lock.h" #if defined(PY_HAVE_JIT_GDB_UNWIND) # include "jit_unwind_info.h" # if !JIT_UNWIND_INFO_SUPPORTED # error "JIT unwind info was not generated for this target" # endif #endif #if defined(PY_HAVE_PERF_TRAMPOLINE) \ || defined(PY_HAVE_JIT_GDB_UNWIND) \ || defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) #if defined(PY_HAVE_JIT_GDB_UNWIND) # include <elf.h> #endif #if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) /* * libgcc exposes frame registration entry points, but GCC's public headers * on some distributions do not declare them even though the symbols are * available in libgcc_s. */ void __register_frame(const void *); void __deregister_frame(const void *); #endif #include <stdio.h> #include <string.h> // ============================================================================= // DWARF CONSTANTS // ============================================================================= /* * DWARF (Debug With Arbitrary Record Formats) constants * * DWARF is a debugging data format used to provide stack unwinding information. * These constants define the various encoding types and opcodes used in * DWARF Call Frame Information (CFI) records. */ /* DWARF Call Frame Information version */ #define DWRF_CIE_VERSION 1 /* DWARF CFA (Call Frame Address) opcodes */ enum { DWRF_CFA_nop = 0x0, // No operation DWRF_CFA_offset_extended = 0x5, // Extended offset instruction DWRF_CFA_def_cfa = 0xc, // Define CFA rule DWRF_CFA_def_cfa_register = 0xd, // Define CFA register DWRF_CFA_def_cfa_offset = 0xe, // Define CFA offset DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset DWRF_CFA_advance_loc = 0x40, // Advance location counter DWRF_CFA_offset = 0x80, // Simple offset instruction #if defined(__aarch64__) DWRF_CFA_AARCH64_negate_ra_state = 0x2d, // Toggle return address signing state #endif DWRF_CFA_restore = 0xc0 // Restore register }; /* * Architecture-specific DWARF register numbers * * These constants define the register numbering scheme used by DWARF * for each supported architecture. The numbers must match the ABI * specification for proper stack unwinding. */ enum { #ifdef __x86_64__ /* x86_64 register numbering (note: order is defined by x86_64 ABI) */ DWRF_REG_AX, // RAX DWRF_REG_DX, // RDX DWRF_REG_CX, // RCX DWRF_REG_BX, // RBX DWRF_REG_SI, // RSI DWRF_REG_DI, // RDI DWRF_REG_BP, // RBP DWRF_REG_SP, // RSP DWRF_REG_8, // R8 DWRF_REG_9, // R9 DWRF_REG_10, // R10 DWRF_REG_11, // R11 DWRF_REG_12, // R12 DWRF_REG_13, // R13 DWRF_REG_14, // R14 DWRF_REG_15, // R15 DWRF_REG_RA, // Return address (RIP) #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) /* AArch64 register numbering */ DWRF_REG_FP = 29, // Frame Pointer DWRF_REG_RA = 30, // Link register (return address) DWRF_REG_SP = 31, // Stack pointer #else # error "Unsupported target architecture" #endif }; // ============================================================================= // ELF OBJECT CONTEXT // ============================================================================= /* * Context for building ELF/DWARF structures * * This structure maintains state while constructing DWARF unwind information. * It acts as a simple buffer manager with pointers to track current position * and important landmarks within the buffer. */ typedef struct ELFObjectContext { uint8_t* p; // Current write position in buffer uint8_t* startp; // Start of buffer (for offset calculations) uint8_t* fde_p; // Start of FDE data (for PC-relative calculations) uintptr_t code_addr; // Address of the code section size_t code_size; // Size of the code section } ELFObjectContext; // ============================================================================= // DWARF GENERATION UTILITIES // ============================================================================= /* * Append a null-terminated string to the ELF context buffer. * * Args: * ctx: ELF object context * str: String to append (must be null-terminated) * * Returns: Offset from start of buffer where string was written */ static uint32_t elfctx_append_string(ELFObjectContext* ctx, const char* str) { uint8_t* p = ctx->p; uint32_t ofs = (uint32_t)(p - ctx->startp); /* Copy string including null terminator */ do { *p++ = (uint8_t)*str; } while (*str++); ctx->p = p; return ofs; } /* * Append a SLEB128 (Signed Little Endian Base 128) value * * SLEB128 is a variable-length encoding used extensively in DWARF. * It efficiently encodes small numbers in fewer bytes. * * Args: * ctx: ELF object context * v: Signed value to encode */ static void elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) { uint8_t* p = ctx->p; /* Encode 7 bits at a time, with continuation bit in MSB */ for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) { *p++ = (uint8_t)((v & 0x7f) | 0x80); // Set continuation bit } *p++ = (uint8_t)(v & 0x7f); // Final byte without continuation bit ctx->p = p; } /* * Append a ULEB128 (Unsigned Little Endian Base 128) value * * Similar to SLEB128 but for unsigned values. * * Args: * ctx: ELF object context * v: Unsigned value to encode */ static void elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) { uint8_t* p = ctx->p; /* Encode 7 bits at a time, with continuation bit in MSB */ for (; v >= 0x80; v >>= 7) { *p++ = (char)((v & 0x7f) | 0x80); // Set continuation bit } *p++ = (char)v; // Final byte without continuation bit ctx->p = p; } /* * Macros for generating DWARF structures * * These macros provide a convenient way to write various data types * to the DWARF buffer while automatically advancing the pointer. */ #define DWRF_U8(x) (*p++ = (x)) // Write unsigned 8-bit #define DWRF_I8(x) (*(int8_t*)p = (x), p++) // Write signed 8-bit #define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) // Write unsigned 16-bit #define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) // Write unsigned 32-bit #define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) // Write address #define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) // Write ULEB128 #define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) // Write SLEB128 #define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) // Write string /* Align to specified boundary with NOP instructions */ #define DWRF_ALIGNNOP(s) \ while ((uintptr_t)p & ((s)-1)) { \ *p++ = DWRF_CFA_nop; \ } /* Write a DWARF section with automatic size calculation */ #define DWRF_SECTION(name, stmt) \ { \ uint32_t* szp_##name = (uint32_t*)p; \ p += 4; \ stmt; \ *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ } // ============================================================================= // DWARF EH FRAME GENERATION // ============================================================================= static void elf_init_ehframe_perf(ELFObjectContext* ctx); #if defined(PY_HAVE_JIT_GDB_UNWIND) static void elf_init_ehframe_gdb(ELFObjectContext* ctx); #endif static inline void elf_init_ehframe(ELFObjectContext* ctx, int absolute_addr) { if (absolute_addr) { #if defined(PY_HAVE_JIT_GDB_UNWIND) elf_init_ehframe_gdb(ctx); #else Py_UNREACHABLE(); #endif } else { elf_init_ehframe_perf(ctx); } } size_t _PyJitUnwind_EhFrameSize(int absolute_addr) { /* The .eh_frame we emit is small and bounded; keep a generous buffer. */ uint8_t scratch[512]; _Static_assert(sizeof(scratch) >= 256, "scratch buffer may be too small for elf_init_ehframe"); ELFObjectContext ctx; ctx.code_size = 1; ctx.code_addr = 0; ctx.startp = ctx.p = scratch; ctx.fde_p = NULL; /* Generate once into scratch to learn the required size. */ elf_init_ehframe(&ctx, absolute_addr); ptrdiff_t size = ctx.p - ctx.startp; assert(size <= (ptrdiff_t)sizeof(scratch)); return (size_t)size; } size_t _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, const void *code_addr, size_t code_size, int absolute_addr) { if (buffer == NULL || code_addr == NULL || code_size == 0) { return 0; } /* Generate the frame twice: once to size-check, once to write. */ size_t required = _PyJitUnwind_EhFrameSize(absolute_addr); if (required == 0 || required > buffer_size) { return 0; } ELFObjectContext ctx; ctx.code_size = code_size; ctx.code_addr = (uintptr_t)code_addr; ctx.startp = ctx.p = buffer; ctx.fde_p = NULL; elf_init_ehframe(&ctx, absolute_addr); size_t written = (size_t)(ctx.p - ctx.startp); /* The frame size is independent of code_addr/code_size (fixed-width fields). */ assert(written == required); return written; } /* * Generate a minimal .eh_frame for a single JIT code region. * * The .eh_frame section contains Call Frame Information (CFI) that describes * how to unwind the stack at any point in the code. This is essential for * unwinding through JIT-generated code. * * The generated data contains: * 1. A CIE (Common Information Entry) describing the calling convention. * 2. An FDE (Frame Description Entry) describing how to unwind the JIT frame. * * Two flavors are emitted, dispatched on the absolute_addr flag: * * - absolute_addr == 0 (elf_init_ehframe_perf): PC-relative FDE address * encoding for perf's synthesized DSO layout. The CIE describes the * trampoline's entry state and the FDE walks through the prologue and * epilogue with advance_loc instructions. This matches the pre-existing * perf_jit_trampoline behavior byte-for-byte. * * - absolute_addr == 1 (elf_init_ehframe_gdb): absolute FDE address * encoding for the GDB JIT in-memory ELF. The CIE describes the * steady-state frame layout (CFA = %rbp+16 / x29+16, with saved fp and * return-address column at fixed offsets) and the FDE emits no further * CFI. The same rule applies at every PC in the registered region, * which is correct for executor stencils (they pin the frame pointer * across the region). This is the GDB-side fix; see elf_init_ehframe_gdb * for details. */ static void elf_init_ehframe_perf(ELFObjectContext* ctx) { int fde_ptr_enc = DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4; uint8_t* p = ctx->p; uint8_t* framep = p; // Remember start of frame data /* * DWARF Unwind Table for Trampoline Function * * This section defines DWARF Call Frame Information (CFI) using encoded macros * like `DWRF_U8`, `DWRF_UV`, and `DWRF_SECTION` to describe how the trampoline function * preserves and restores registers. This is used by profiling tools (e.g., `perf`) * and debuggers for stack unwinding in JIT-compiled code. * * ------------------------------------------------- * TO REGENERATE THIS TABLE FROM GCC OBJECTS: * ------------------------------------------------- * * 1. Create a trampoline source file (e.g., `trampoline.c`): * * #include <Python.h> * typedef PyObject* (*py_evaluator)(void*, void*, int); * PyObject* trampoline(void *ts, void *f, int throwflag, py_evaluator evaluator) { * return evaluator(ts, f, throwflag); * } * * 2. Compile to an object file with frame pointer preservation: * * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c * * 3. Extract DWARF unwind info from the object file: * * readelf -w trampoline.o * * Example output from `.eh_frame`: * * 00000000 CIE * Version: 1 * Augmentation: "zR" * Code alignment factor: 4 * Data alignment factor: -8 * Return address column: 30 * DW_CFA_def_cfa: r31 (sp) ofs 0 * * 00000014 FDE cie=00000000 pc=0..14 * DW_CFA_advance_loc: 4 * DW_CFA_def_cfa_offset: 16 * DW_CFA_offset: r29 at cfa-16 * DW_CFA_offset: r30 at cfa-8 * DW_CFA_advance_loc: 12 * DW_CFA_restore: r30 * DW_CFA_restore: r29 * DW_CFA_def_cfa_offset: 0 * * -- These values can be verified by comparing with `readelf -w` or `llvm-dwarfdump --eh-frame`. * * ---------------------------------- * HOW TO TRANSLATE TO DWRF_* MACROS: * ---------------------------------- * * After compiling your trampoline with: * * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c * * run: * * readelf -w trampoline.o * * to inspect the generated `.eh_frame` data. You will see two main components: * * 1. A CIE (Common Information Entry): shared configuration used by all FDEs. * 2. An FDE (Frame Description Entry): function-specific unwind instructions. * * --------------------- * Translating the CIE: * --------------------- * From `readelf -w`, you might see: * * 00000000 0000000000000010 00000000 CIE * Version: 1 * Augmentation: "zR" * Code alignment factor: 4 * Data alignment factor: -8 * Return address column: 30 * Augmentation data: 1b * DW_CFA_def_cfa: r31 (sp) ofs 0 * * Map this to: * * DWRF_SECTION(CIE, * DWRF_U32(0); // CIE ID (always 0 for CIEs) * DWRF_U8(DWRF_CIE_VERSION); // Version: 1 * DWRF_STR("zR"); // Augmentation string "zR" * DWRF_UV(4); // Code alignment factor = 4 * DWRF_SV(-8); // Data alignment factor = -8 * DWRF_U8(DWRF_REG_RA); // Return address register (e.g., x30 = 30) * DWRF_UV(1); // Augmentation data length = 1 * DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); // Encoding for FDE pointers * * DWRF_U8(DWRF_CFA_def_cfa); // DW_CFA_def_cfa * DWRF_UV(DWRF_REG_SP); // Register: SP (r31) * DWRF_UV(0); // Offset = 0 * * DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer size boundary * ) * * Notes: * - Use `DWRF_UV` for unsigned LEB128, `DWRF_SV` for signed LEB128. * - `DWRF_REG_RA` and `DWRF_REG_SP` are architecture-defined constants. * * --------------------- * Translating the FDE: * --------------------- * From `readelf -w`: * * 00000014 0000000000000020 00000018 FDE cie=00000000 pc=0000000000000000..0000000000000014 * DW_CFA_advance_loc: 4 * DW_CFA_def_cfa_offset: 16 * DW_CFA_offset: r29 at cfa-16 * DW_CFA_offset: r30 at cfa-8 * DW_CFA_advance_loc: 12 * DW_CFA_restore: r30 * DW_CFA_restore: r29 * DW_CFA_def_cfa_offset: 0 * * Map the FDE header and instructions to: * * DWRF_SECTION(FDE, * DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (relative from here) * DWRF_U32(pc_relative_offset); // PC-relative location of the code (calculated dynamically) * DWRF_U32(ctx->code_size); // Code range covered by this FDE * DWRF_U8(0); // Augmentation data length (none) * * DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance location by 1 unit (1 * 4 = 4 bytes) * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 * DWRF_UV(16); * * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Save x29 (frame pointer) * DWRF_UV(2); // At offset 2 * 8 = 16 bytes * * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Save x30 (return address) * DWRF_UV(1); // At offset 1 * 8 = 8 bytes * * DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance location by 3 units (3 * 4 = 12 bytes) * * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Restore x30 * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Restore x29 * * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP * DWRF_UV(0); * ) * * To regenerate: * 1. Get the `code alignment factor`, `data alignment factor`, and `RA column` from the CIE. * 2. Note the range of the function from the FDE's `pc=...` line and map it to the JIT code as * the code is in a different address space every time. * 3. For each `DW_CFA_*` entry, use the corresponding `DWRF_*` macro: * - `DW_CFA_def_cfa_offset` → DWRF_U8(DWRF_CFA_def_cfa_offset), DWRF_UV(value) * - `DW_CFA_offset: rX` → DWRF_U8(DWRF_CFA_offset | reg), DWRF_UV(offset) * - `DW_CFA_restore: rX` → DWRF_U8(DWRF_CFA_offset | reg) // restore is same as reusing offset * - `DW_CFA_advance_loc: N` → DWRF_U8(DWRF_CFA_advance_loc | (N / code_alignment_factor)) * 4. Use `DWRF_REG_FP`, `DWRF_REG_RA`, etc., for register numbers. * 5. Use `sizeof(uintptr_t)` (typically 8) for pointer size calculations and alignment. */ /* * Emit DWARF EH CIE (Common Information Entry) * * The CIE describes the calling conventions and basic unwinding rules * that apply to all functions in this compilation unit. */ DWRF_SECTION(CIE, DWRF_U32(0); // CIE ID (0 indicates this is a CIE) DWRF_U8(DWRF_CIE_VERSION); // CIE version (1) DWRF_STR("zR"); // Augmentation string ("zR" = has LSDA) #ifdef __x86_64__ DWRF_UV(1); // Code alignment factor (x86_64: 1 byte) #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) DWRF_UV(4); // Code alignment factor (AArch64: 4 bytes per instruction) #endif DWRF_SV(-(int64_t)sizeof(uintptr_t)); // Data alignment factor (negative) DWRF_U8(DWRF_REG_RA); // Return address register number DWRF_UV(1); // Augmentation data length DWRF_U8(fde_ptr_enc); // FDE pointer encoding /* Initial CFI instructions - describe default calling convention */ #ifdef __x86_64__ /* x86_64 initial CFI state */ DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) DWRF_UV(DWRF_REG_SP); // CFA = SP register DWRF_UV(sizeof(uintptr_t)); // CFA = SP + pointer_size DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); // Return address is saved DWRF_UV(1); // At offset 1 from CFA #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) /* AArch64 initial CFI state */ DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) DWRF_UV(DWRF_REG_SP); // CFA = SP register DWRF_UV(0); // CFA = SP + 0 (AArch64 starts with offset 0) // No initial register saves in AArch64 CIE #endif DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary ) /* * Emit DWARF EH FDE (Frame Description Entry) * * The FDE describes unwinding information specific to this function. * It references the CIE and provides function-specific CFI instructions. * * The PC-relative offset is calculated after the entire EH frame is built * to ensure accurate positioning relative to the synthesized DSO layout. */ DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (backwards reference) /* * In perf jitdump mode the FDE PC field is encoded PC-relative and * points back to code_start. Record where that field lives so we can * patch in the final offset after the rest of the synthetic DSO * layout is known. */ ctx->fde_p = p; // Remember where PC offset field is located for later calculation DWRF_U32(0); // Placeholder for PC-relative offset (calculated below) DWRF_U32(ctx->code_size); // Address range covered by this FDE (code length) DWRF_U8(0); // Augmentation data length (none) /* * Architecture-specific CFI instructions * * These instructions describe how registers are saved and restored * during function calls. Each architecture has different calling * conventions and register usage patterns. */ #ifdef __x86_64__ /* x86_64 calling convention unwinding rules */ # if defined(__CET__) && (__CET__ & 1) DWRF_U8(DWRF_CFA_advance_loc | 4); // Advance past endbr64 (4 bytes) # endif DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past push %rbp (1 byte) DWRF_U8(DWRF_CFA_def_cfa_offset); // def_cfa_offset 16 DWRF_UV(16); // New offset: SP + 16 DWRF_U8(DWRF_CFA_offset | DWRF_REG_BP); // offset r6 at cfa-16 DWRF_UV(2); // Offset factor: 2 * 8 = 16 bytes DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past mov %rsp,%rbp (3 bytes) DWRF_U8(DWRF_CFA_def_cfa_register); // def_cfa_register r6 DWRF_UV(DWRF_REG_BP); // Use base pointer register DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past call *%rcx (2 bytes) + pop %rbp (1 byte) = 3 DWRF_U8(DWRF_CFA_def_cfa); // def_cfa r7 ofs 8 DWRF_UV(DWRF_REG_SP); // Use stack pointer register DWRF_UV(8); // New offset: SP + 8 #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) /* AArch64 calling convention unwinding rules */ #if defined(__ARM_FEATURE_PAC_DEFAULT) || \ (defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1) DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past SIGN_LR (4 bytes) #endif #if defined(__ARM_FEATURE_PAC_DEFAULT) DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // Saved LR is PAC-signed from here #endif DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 DWRF_UV(16); // Stack pointer moved by 16 bytes DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // x29 (frame pointer) saved DWRF_UV(2); // At CFA-16 (2 * 8 = 16 bytes from CFA) DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) #if defined(__ARM_FEATURE_PAC_DEFAULT) DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // LR is authenticated, no longer PAC-signed #endif DWRF_U8(DWRF_CFA_def_cfa_register); // CFA = FP (x29) + 16 DWRF_UV(DWRF_REG_FP); DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! DWRF_U8(DWRF_CFA_restore | DWRF_REG_FP); // Restore x29 - NO DWRF_UV() after this! DWRF_U8(DWRF_CFA_def_cfa); // CFA = SP + 0 (stack restored) DWRF_UV(DWRF_REG_SP); DWRF_UV(0); #else # error "Unsupported target architecture" #endif DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary ) ctx->p = p; // Update context pointer to end of generated data /* Calculate and update the PC-relative offset in the FDE * * When perf processes the jitdump, it creates a synthesized DSO with this layout: * * Synthesized DSO Memory Layout: * ┌─────────────────────────────────────────────────────────────┐ < code_start * │ Code Section │ * │ (round_up(code_size, 8) bytes) │ * ├─────────────────────────────────────────────────────────────┤ < start of EH frame data * │ EH Frame Data │ * │ ┌─────────────────────────────────────────────────────┐ │ * │ │ CIE data │ │ * │ └─────────────────────────────────────────────────────┘ │ * │ ┌─────────────────────────────────────────────────────┐ │ * │ │ FDE Header: │ │ * │ │ - CIE offset (4 bytes) │ │ * │ │ - PC offset (4 bytes) <─ fde_offset_in_frame ─────┼────┼─> points to code_start * │ │ - address range (4 bytes) │ │ (this specific field) * │ │ CFI Instructions... │ │ * │ └─────────────────────────────────────────────────────┘ │ * ├─────────────────────────────────────────────────────────────┤ < reference_point * │ EhFrameHeader │ * │ (navigation metadata) │ * └─────────────────────────────────────────────────────────────┘ * * The PC offset field in the FDE must contain the distance from itself to code_start: * * distance = code_start - fde_pc_field * * Where: * fde_pc_field_location = reference_point - eh_frame_size + fde_offset_in_frame * code_start_location = reference_point - eh_frame_size - round_up(code_size, 8) * * Therefore: * distance = code_start_location - fde_pc_field_location * = (ref - eh_frame_size - rounded_code_size) - (ref - eh_frame_size + fde_offset_in_frame) * = -rounded_code_size - fde_offset_in_frame * = -(round_up(code_size, 8) + fde_offset_in_frame) * * Note: fde_offset_in_frame is the offset from EH frame start to the PC offset field. * */ int32_t rounded_code_size = (int32_t)_Py_SIZE_ROUND_UP(ctx->code_size, 8); int32_t fde_offset_in_frame = (int32_t)(ctx->fde_p - framep); *(int32_t *)ctx->fde_p = -(rounded_code_size + fde_offset_in_frame); } /* * Build .eh_frame data for the GDB JIT interface. * * The executor runs inside the frame established by _PyJIT_Entry, but the * synthetic executor FDE collapses that state into a single logical JIT frame * that unwinds directly into _PyEval_*. Executor stencils never touch the * frame pointer - enforced by Tools/jit/_optimizers.py _validate() and * -mframe-pointer=reserved - so the steady-state rule is valid at every PC * and the FDE body is empty. Tools/jit/_targets.py derives the initial CFI * rules from the row active at the executor call in the compiled shim object. */ #if defined(PY_HAVE_JIT_GDB_UNWIND) static void elf_init_ehframe_gdb(ELFObjectContext* ctx) { int fde_ptr_enc = DWRF_EH_PE_absptr; uint8_t* p = ctx->p; uint8_t* framep = p; DWRF_SECTION(CIE, DWRF_U32(0); // CIE ID DWRF_U8(DWRF_CIE_VERSION); DWRF_STR("zR"); // aug data length + FDE ptr encoding follow DWRF_UV(JIT_UNWIND_CODE_ALIGNMENT_FACTOR); DWRF_SV(JIT_UNWIND_DATA_ALIGNMENT_FACTOR); DWRF_U8(JIT_UNWIND_RA_REG); DWRF_UV(1); // Augmentation data length DWRF_U8(fde_ptr_enc); // FDE pointer encoding /* Executor steady-state rule (our invariant, not the compiler's). */ DWRF_U8(DWRF_CFA_def_cfa); DWRF_UV(JIT_UNWIND_CFA_REG); DWRF_UV(JIT_UNWIND_CFA_OFFSET); DWRF_U8(DWRF_CFA_offset | JIT_UNWIND_FP_REG); DWRF_UV(JIT_UNWIND_FP_OFFSET); DWRF_U8(DWRF_CFA_offset | JIT_UNWIND_RA_REG); DWRF_UV(JIT_UNWIND_RA_OFFSET); DWRF_ALIGNNOP(sizeof(uintptr_t)); ) DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (backwards reference) DWRF_ADDR(ctx->code_addr); // Absolute code start DWRF_ADDR((uintptr_t)ctx->code_size); // Code range covered DWRF_U8(0); // Augmentation data length (none) DWRF_ALIGNNOP(sizeof(uintptr_t)); ) ctx->p = p; } #endif #if defined(PY_HAVE_JIT_GDB_UNWIND) enum { JIT_NOACTION = 0, JIT_REGISTER_FN = 1, JIT_UNREGISTER_FN = 2, }; struct jit_code_entry { struct jit_code_entry *next; struct jit_code_entry *prev; const char *symfile_addr; uint64_t symfile_size; const void *code_addr; }; struct jit_descriptor { uint32_t version; uint32_t action_flag; struct jit_code_entry *relevant_entry; struct jit_code_entry *first_entry; }; PyMutex _Py_jit_debug_mutex = {0}; Py_EXPORTED_SYMBOL volatile struct jit_descriptor __jit_debug_descriptor = { 1, JIT_NOACTION, NULL, NULL }; Py_EXPORTED_SYMBOL void __attribute__((noinline)) __jit_debug_register_code(void) { /* Keep this call visible to debuggers and not optimized away. */ (void)__jit_debug_descriptor.action_flag; #if defined(__GNUC__) || defined(__clang__) __asm__ __volatile__("" ::: "memory"); #endif } static uint16_t gdb_jit_machine_id(void) { /* Map the current target to ELF e_machine; return 0 to skip registration. */ #if defined(__x86_64__) || defined(_M_X64) return EM_X86_64; #elif defined(__aarch64__) && !defined(__ILP32__) return EM_AARCH64; #else return 0; #endif } static struct jit_code_entry * gdb_jit_register_code( const void *code_addr, size_t code_size, const char *symname, const uint8_t *eh_frame, size_t eh_frame_size ) { /* * Build a minimal in-memory ELF for GDB's JIT interface and link it into * __jit_debug_descriptor so debuggers can resolve JIT code. */ if (code_addr == NULL || code_size == 0 || symname == NULL) { return NULL; } const uint16_t machine = gdb_jit_machine_id(); if (machine == 0) { return NULL; } enum { SH_NULL = 0, SH_TEXT, SH_EH_FRAME, SH_SHSTRTAB, SH_STRTAB, SH_SYMTAB, SH_NUM, }; static const char shstrtab[] = "\0.text\0.eh_frame\0.shstrtab\0.strtab\0.symtab"; _Static_assert(sizeof(shstrtab) == 1 + sizeof(".text") + sizeof(".eh_frame") + sizeof(".shstrtab") + sizeof(".strtab") + sizeof(".symtab"), "shstrtab size mismatch"); const size_t shstrtab_size = sizeof(shstrtab); const size_t sh_text = 1; const size_t sh_eh_frame = sh_text + sizeof(".text"); const size_t sh_shstrtab = sh_eh_frame + sizeof(".eh_frame"); const size_t sh_strtab = sh_shstrtab + sizeof(".shstrtab"); const size_t sh_symtab = sh_strtab + sizeof(".strtab"); const size_t text_size = code_size; const size_t text_padded = _Py_SIZE_ROUND_UP(text_size, 8); const size_t strtab_size = 1 + strlen(symname) + 1; const size_t symtab_size = 3 * sizeof(Elf64_Sym); size_t offset = sizeof(Elf64_Ehdr); offset = _Py_SIZE_ROUND_UP(offset, 16); const size_t text_off = offset; const size_t eh_off = text_off + text_padded; offset = eh_off + eh_frame_size; const size_t shstr_off = offset; offset += shstrtab_size; const size_t str_off = offset; offset += strtab_size; /* Elf64_Sym requires 8-byte alignment for st_value/st_size. */ offset = _Py_SIZE_ROUND_UP(offset, 8); const size_t sym_off = offset; offset += symtab_size; offset = _Py_SIZE_ROUND_UP(offset, sizeof(Elf64_Shdr)); const size_t sh_off = offset; const size_t shnum = SH_NUM; const size_t total_size = sh_off + shnum * sizeof(Elf64_Shdr); uint8_t *buf = (uint8_t *)PyMem_RawMalloc(total_size); if (buf == NULL) { return NULL; } memset(buf, 0, total_size); Elf64_Ehdr *ehdr = (Elf64_Ehdr *)buf; memcpy(ehdr->e_ident, ELFMAG, SELFMAG); ehdr->e_ident[EI_CLASS] = ELFCLASS64; ehdr->e_ident[EI_DATA] = ELFDATA2LSB; ehdr->e_ident[EI_VERSION] = EV_CURRENT; ehdr->e_ident[EI_OSABI] = ELFOSABI_NONE; ehdr->e_type = ET_DYN; ehdr->e_machine = machine; ehdr->e_version = EV_CURRENT; ehdr->e_entry = 0; ehdr->e_phoff = 0; ehdr->e_shoff = sh_off; ehdr->e_ehsize = sizeof(Elf64_Ehdr); ehdr->e_shentsize = sizeof(Elf64_Shdr); ehdr->e_shnum = shnum; ehdr->e_shstrndx = SH_SHSTRTAB; memcpy(buf + text_off, code_addr, text_size); memcpy(buf + eh_off, eh_frame, eh_frame_size); char *shstr = (char *)(buf + shstr_off); memcpy(shstr, shstrtab, shstrtab_size); char *strtab = (char *)(buf + str_off); strtab[0] = '\0'; memcpy(strtab + 1, symname, strlen(symname)); strtab[strtab_size - 1] = '\0'; Elf64_Sym *syms = (Elf64_Sym *)(buf + sym_off); memset(syms, 0, symtab_size); /* Section symbol for .text (local) */ syms[1].st_info = ELF64_ST_INFO(STB_LOCAL, STT_SECTION); syms[1].st_shndx = 1; /* Function symbol */ syms[2].st_name = 1; syms[2].st_info = ELF64_ST_INFO(STB_GLOBAL, STT_FUNC); syms[2].st_other = STV_DEFAULT; syms[2].st_shndx = 1; /* For ET_DYN/ET_EXEC, st_value is the absolute virtual address. */ syms[2].st_value = (Elf64_Addr)(uintptr_t)code_addr; syms[2].st_size = code_size; Elf64_Shdr *shdrs = (Elf64_Shdr *)(buf + sh_off); memset(shdrs, 0, shnum * sizeof(Elf64_Shdr)); shdrs[SH_TEXT].sh_name = sh_text; shdrs[SH_TEXT].sh_type = SHT_PROGBITS; shdrs[SH_TEXT].sh_flags = SHF_ALLOC | SHF_EXECINSTR; shdrs[SH_TEXT].sh_addr = (Elf64_Addr)(uintptr_t)code_addr; shdrs[SH_TEXT].sh_offset = text_off; shdrs[SH_TEXT].sh_size = text_size; shdrs[SH_TEXT].sh_addralign = 16; shdrs[SH_EH_FRAME].sh_name = sh_eh_frame; shdrs[SH_EH_FRAME].sh_type = SHT_PROGBITS; shdrs[SH_EH_FRAME].sh_flags = SHF_ALLOC; shdrs[SH_EH_FRAME].sh_addr = (Elf64_Addr)((uintptr_t)code_addr + text_padded); shdrs[SH_EH_FRAME].sh_offset = eh_off; shdrs[SH_EH_FRAME].sh_size = eh_frame_size; shdrs[SH_EH_FRAME].sh_addralign = 8; shdrs[SH_SHSTRTAB].sh_name = sh_shstrtab; shdrs[SH_SHSTRTAB].sh_type = SHT_STRTAB; shdrs[SH_SHSTRTAB].sh_offset = shstr_off; shdrs[SH_SHSTRTAB].sh_size = shstrtab_size; shdrs[SH_SHSTRTAB].sh_addralign = 1; shdrs[SH_STRTAB].sh_name = sh_strtab; shdrs[SH_STRTAB].sh_type = SHT_STRTAB; shdrs[SH_STRTAB].sh_offset = str_off; shdrs[SH_STRTAB].sh_size = strtab_size; shdrs[SH_STRTAB].sh_addralign = 1; shdrs[SH_SYMTAB].sh_name = sh_symtab; shdrs[SH_SYMTAB].sh_type = SHT_SYMTAB; shdrs[SH_SYMTAB].sh_offset = sym_off; shdrs[SH_SYMTAB].sh_size = symtab_size; shdrs[SH_SYMTAB].sh_link = SH_STRTAB; shdrs[SH_SYMTAB].sh_info = 2; shdrs[SH_SYMTAB].sh_addralign = 8; shdrs[SH_SYMTAB].sh_entsize = sizeof(Elf64_Sym); struct jit_code_entry *entry = PyMem_RawMalloc(sizeof(*entry)); if (entry == NULL) { PyMem_RawFree(buf); return NULL; } entry->symfile_addr = (const char *)buf; entry->symfile_size = total_size; entry->code_addr = code_addr; PyMutex_Lock(&_Py_jit_debug_mutex); entry->prev = NULL; entry->next = __jit_debug_descriptor.first_entry; if (entry->next != NULL) { entry->next->prev = entry; } __jit_debug_descriptor.first_entry = entry; __jit_debug_descriptor.relevant_entry = entry; __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; __jit_debug_register_code(); __jit_debug_descriptor.action_flag = JIT_NOACTION; __jit_debug_descriptor.relevant_entry = NULL; PyMutex_Unlock(&_Py_jit_debug_mutex); return entry; } #endif // defined(PY_HAVE_JIT_GDB_UNWIND) void * _PyJitUnwind_GdbRegisterCode(const void *code_addr, size_t code_size, const char *entry, const char *filename) { #if defined(PY_HAVE_JIT_GDB_UNWIND) /* GDB expects a stable symbol name and absolute addresses in .eh_frame. */ if (entry == NULL) { entry = ""; } if (filename == NULL) { filename = ""; } size_t name_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1; char *name = (char *)PyMem_RawMalloc(name_size); if (name == NULL) { return NULL; } snprintf(name, name_size, "py::%s:%s", entry, filename); uint8_t buffer[1024]; size_t eh_frame_size = _PyJitUnwind_BuildEhFrame( buffer, sizeof(buffer), code_addr, code_size, 1); if (eh_frame_size == 0) { PyMem_RawFree(name); return NULL; } void *handle = gdb_jit_register_code(code_addr, code_size, name, buffer, eh_frame_size); PyMem_RawFree(name); return handle; #else (void)code_addr; (void)code_size; (void)entry; (void)filename; return NULL; #endif } void _PyJitUnwind_GdbUnregisterCode(void *handle) { #if defined(PY_HAVE_JIT_GDB_UNWIND) struct jit_code_entry *entry = (struct jit_code_entry *)handle; if (entry == NULL) { return; } PyMutex_Lock(&_Py_jit_debug_mutex); if (entry->prev != NULL) { entry->prev->next = entry->next; } else { __jit_debug_descriptor.first_entry = entry->next; } if (entry->next != NULL) { entry->next->prev = entry->prev; } __jit_debug_descriptor.relevant_entry = entry; __jit_debug_descriptor.action_flag = JIT_UNREGISTER_FN; __jit_debug_register_code(); __jit_debug_descriptor.action_flag = JIT_NOACTION; __jit_debug_descriptor.relevant_entry = NULL; |
| You are visitor 01402503 — GeezerVison does not store the pages you look up. Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required. |