| 358 | } |
| 359 | |
| 360 | TaskTracer::Result TaskTracer::SignalTrace(pthread_t worker_tid) { |
| 361 | // CAUTION: |
| 362 | // https://github.com/gperftools/gperftools/wiki/gperftools'-stacktrace-capturing-methods-and-their-issues#libunwind |
| 363 | // Generally, libunwind promises async-signal-safety for capturing backtraces. |
| 364 | // But in practice, it is only partially async-signal-safe due to reliance on |
| 365 | // dl_iterate_phdr API, which is used to enumerate all loaded ELF modules |
| 366 | // (.so files and main executable binary). No libc offers dl_iterate_pdhr that |
| 367 | // is async-signal-safe. In practice, the issue may happen if we take tracing |
| 368 | // signal during an existing dl_iterate_phdr call (such as when the program |
| 369 | // throws an exception) or during dlopen/dlclose-ing some .so module. |
| 370 | // Deadlock call stack: |
| 371 | // #0 __lll_lock_wait (futex=futex@entry=0x7f0d3d7f0990 <_rtld_global+2352>, private=0) at lowlevellock.c:52 |
| 372 | // #1 0x00007f0d3a73c131 in __GI___pthread_mutex_lock (mutex=0x7f0d3d7f0990 <_rtld_global+2352>) at ../nptl/pthread_mutex_lock.c:115 |
| 373 | // #2 0x00007f0d38eb0231 in __GI___dl_iterate_phdr (callback=callback@entry=0x7f0d38c456a0 <_ULx86_64_dwarf_callback>, data=data@entry=0x7f0d07defad0) at dl-iteratephdr.c:40 |
| 374 | // #3 0x00007f0d38c45d79 in _ULx86_64_dwarf_find_proc_info (as=0x7f0d38c4f340 <local_addr_space>, ip=ip@entry=139694791966897, pi=pi@entry=0x7f0d07df0498, need_unwind_info=need_unwind_info@entry=1, arg=0x7f0d07df0340) at dwarf/Gfind_proc_info-lsb.c:759 |
| 375 | // #4 0x00007f0d38c43260 in fetch_proc_info (c=c@entry=0x7f0d07df0340, ip=139694791966897) at dwarf/Gparser.c:461 |
| 376 | // #5 0x00007f0d38c44e46 in find_reg_state (sr=0x7f0d07defd10, c=0x7f0d07df0340) at dwarf/Gparser.c:925 |
| 377 | // #6 _ULx86_64_dwarf_step (c=c@entry=0x7f0d07df0340) at dwarf/Gparser.c:972 |
| 378 | // #7 0x00007f0d38c40c14 in _ULx86_64_step (cursor=cursor@entry=0x7f0d07df0340) at x86_64/Gstep.c:71 |
| 379 | // #8 0x00007f0d399ed8f6 in GetStackTraceWithContext_libunwind (result=<optimized out>, max_depth=63, skip_count=132054887, ucp=<optimized out>) at src/stacktrace_libunwind-inl.h:138 |
| 380 | // #9 0x00007f0d399ee083 in GetStackTraceWithContext (result=0x7f0d07df07b8, max_depth=63, skip_count=3, uc=0x7f0d07df0a40) at src/stacktrace.cc:305 |
| 381 | // #10 0x00007f0d399ea992 in CpuProfiler::prof_handler (signal_ucontext=<optimized out>, cpu_profiler=0x7f0d399f6600, sig=<optimized out>) at src/profiler.cc:359 |
| 382 | // #11 0x00007f0d399eb633 in ProfileHandler::SignalHandler (sig=27, sinfo=0x7f0d07df0b70, ucontext=0x7f0d07df0a40) at src/profile-handler.cc:530 |
| 383 | // #12 <signal handler called> |
| 384 | // #13 0x00007f0d3a73c0b1 in __GI___pthread_mutex_lock (mutex=0x7f0d3d7f0990 <_rtld_global+2352>) at ../nptl/pthread_mutex_lock.c:115 |
| 385 | // #14 0x00007f0d38eb0231 in __GI___dl_iterate_phdr (callback=0x7f0d38f525f0, data=0x7f0d07df10c0) at dl-iteratephdr.c:40 |
| 386 | // #15 0x00007f0d38f536c1 in _Unwind_Find_FDE () from /lib/x86_64-linux-gnu/libgcc_s.so.1 |
| 387 | // #16 0x00007f0d38f4f868 in ?? () from /lib/x86_64-linux-gnu/libgcc_s.so.1 |
| 388 | // #17 0x00007f0d38f50a20 in ?? () from /lib/x86_64-linux-gnu/libgcc_s.so.1 |
| 389 | // #18 0x00007f0d38f50f99 in _Unwind_RaiseException () from /lib/x86_64-linux-gnu/libgcc_s.so.1 |
| 390 | // #19 0x00007f0d390088dc in __cxa_throw () from /lib/x86_64-linux-gnu/libstdc++.so.6 |
| 391 | // #20 0x00007f0d3b5b2245 in __cxxabiv1::__cxa_throw (thrownException=0x7f0d114ea8c0, type=0x7f0d3d6dd830 <typeinfo for rockset::GRPCError>, destructor=<optimized out>) at /src/folly/folly/experimental/exception_tracer/ExceptionTracerLib.cpp:107 |
| 392 | // |
| 393 | // Therefore, use async-signal-safe absl::DefaultStackUnwinder instead of libunwind. |
| 394 | |
| 395 | if (pthread_equal(worker_tid, pthread_self())) { |
| 396 | return Result::MakeErrorResult("Forbid to trace self"); |
| 397 | } |
| 398 | |
| 399 | // Each signal trace has an independent SignalSync to |
| 400 | // prevent the previous SignalHandler from affecting the new SignalTrace. |
| 401 | butil::intrusive_ptr<SignalSync> signal_sync(new SignalSync()); |
| 402 | if (!signal_sync->Init()) { |
| 403 | return Result::MakeErrorResult("Fail to init SignalSync"); |
| 404 | } |
| 405 | |
| 406 | // Add reference for SignalHandler. |
| 407 | signal_sync->AddRefManually(); |
| 408 | |
| 409 | sigval value{}; |
| 410 | value.sival_ptr = signal_sync.get(); |
| 411 | size_t sigqueue_try = 0; |
| 412 | while (pthread_sigqueue(worker_tid, _signal_num, value) != 0) { |
| 413 | if (errno != EAGAIN || sigqueue_try++ >= 3) { |
| 414 | // Remove reference for SignalHandler. |
| 415 | signal_sync->RemoveRefManually(); |
| 416 | return Result::MakeErrorResult("Fail to pthread_sigqueue: %s", berror()); |
| 417 | } |
nothing calls this directly
no test coverage detected