Internal change.

tcmalloc/deallocation_profiler.cc

@@ -23,17 +23,16 @@
 #include <iterator>
 #include <limits>
 #include <memory>
-#include <new>
 #include <optional>
 #include <type_traits>
 #include <utility>
+#include <vector>
 
 #include "absl/base/attributes.h"
-#include "absl/base/const_init.h"
 #include "absl/base/internal/low_level_alloc.h"
 #include "absl/base/internal/spinlock.h"
 #include "absl/base/internal/sysinfo.h"
-#include "absl/base/macros.h"
+#include "absl/base/nullability.h"
 #include "absl/container/flat_hash_map.h"
 #include "absl/debugging/stacktrace.h"  // for GetStackTrace
 #include "absl/functional/function_ref.h"
@@ -48,6 +47,7 @@
 #include "tcmalloc/internal/sampled_allocation.h"
 #include "tcmalloc/internal_malloc_extension.h"
 #include "tcmalloc/malloc_extension.h"
+#include "tcmalloc/parameters.h"
 #include "tcmalloc/sampler.h"
 #include "tcmalloc/static_vars.h"
 
@@ -380,16 +380,36 @@ class DeallocationProfiler {
   class DeallocationStackTraceTable
       final : public tcmalloc_internal::ProfileBase {
    public:
+    explicit DeallocationStackTraceTable(Mode mode)
+        : mode_(mode),
+          max_events_(
+              mode == Mode::kEventTrace
+                  ? std::max<int32_t>(0, static_cast<int32_t>(
+                                             tcmalloc_internal::Parameters::
+                                                 event_trace_memory_limit() /
+                                             sizeof(DeallocationSampleRecord)))
+                  : 0) {
+      if (mode_ == Mode::kEventTrace) {
+        events_.reserve(max_events_);
+      }
+    }
+
     // We define the dtor to ensure it is placed in the desired text section.
     ~DeallocationStackTraceTable() override = default;
+
     void AddTrace(const DeallocationSampleRecord& alloc_trace,
                   const DeallocationSampleRecord& dealloc_trace);
 
     void Iterate(
         absl::FunctionRef<void(const Profile::Sample&)> func) const override;
 
     ProfileType Type() const override {
-      return tcmalloc::ProfileType::kLifetimes;
+      switch (mode_) {
+        case Mode::kLifetimes:
+          return tcmalloc::ProfileType::kLifetimes;
+        case Mode::kEventTrace:
+          return tcmalloc::ProfileType::kEventTrace;
+      }
     }
 
     std::optional<absl::Time> StartTime() const override { return start_time_; }
@@ -441,12 +461,22 @@ class DeallocationProfiler {
       }
     };
 
+    Mode mode_;
+
+    // Used in kLifetimes mode.
     absl::flat_hash_map<Key, Value, absl::Hash<Key>, std::equal_to<Key>,
                         AllocAdaptor<std::pair<const Key, Value>, MyAllocator>>
         table_;
 
+    // Used in kEventTrace mode.
+    // This is technically a fixed-size container -- we ::reserve capacity at
+    // construction time and truncate traces after reaching max_events_, hence
+    // the lack of low-level allocator.
+    int32_t max_events_ = 0;
+    std::vector<DeallocationSampleRecord> events_;
+
     absl::Time start_time_ = absl::Now();
-    absl::Time stop_time_;
+    absl::Time stop_time_ = absl::InfiniteFuture();
   };
 
   // Keep track of allocations that are in flight
@@ -456,8 +486,9 @@ class DeallocationProfiler {
   std::unique_ptr<DeallocationStackTraceTable> reports_ = nullptr;
 
  public:
-  explicit DeallocationProfiler(DeallocationProfilerList* list) : list_(list) {
-    reports_ = std::make_unique<DeallocationStackTraceTable>();
+  explicit DeallocationProfiler(DeallocationProfilerList* list, Mode mode)
+      : list_(list) {
+    reports_ = std::make_unique<DeallocationStackTraceTable>(mode);
     list_->Add(this);
   }
 
@@ -495,15 +526,20 @@ class DeallocationProfiler {
 
   void ReportFree(tcmalloc_internal::AllocHandle handle) {
     auto it = allocs_.find(handle);
+    DeallocationSampleRecord sample;
 
-    // Handle the case that we observed the deallocation but not the allocation
+    // Handle the (left-censored) case that we observed the deallocation but not
+    // the allocation. Since we only get the handle here, left-censored
+    // deallocations necessarily have depth = 0 and allocated_size = 0.
     if (it == allocs_.end()) {
-      return;
+      sample = {};
+      sample.stack_trace.sampled_alloc_handle = handle;
+      sample.stack_trace.depth = 0;
+    } else {
+      sample = it->second;
+      allocs_.erase(it);
     }
 
-    DeallocationSampleRecord sample = it->second;
-    allocs_.erase(it);
-
     DeallocationSampleRecord deallocation;
     deallocation.stack_trace = sample.stack_trace;
     deallocation.stack_trace.allocation_time = absl::Now();
@@ -592,6 +628,30 @@ void DeallocationProfiler::DeallocationStackTraceTable::StopAndRecord(
 void DeallocationProfiler::DeallocationStackTraceTable::AddTrace(
     const DeallocationSampleRecord& alloc_trace,
     const DeallocationSampleRecord& dealloc_trace) {
+  if (mode_ == Mode::kEventTrace) {
+    // Ensure we can fit up to 2 records (alloc + dealloc) without exceeding
+    // capacity; otherwise silently truncate the trace.
+    if (events_.size() + 2 <= max_events_) {
+      if (alloc_trace.stack_trace.depth > 0) {
+        events_.push_back(alloc_trace);
+      }
+      if (dealloc_trace.stack_trace.depth > 0) {
+        events_.push_back(dealloc_trace);
+        // In-band signal to Iterate() that this is a deallocation event. We can
+        // do this because:
+        // - Matched events propagate the allocated_size via the
+        //   alloc_trace (and the pair can be associated downstream).
+        // - Left-censored events are only passed to ReportFree as
+        //   alloc_handle-s, i.e. we can't know the allocated_size.
+        events_.back().stack_trace.allocated_size = 0;
+      }
+    }
+    return;
+  }
+
+  // Left-censored samples cannot be aggregated with lifetimes
+  if (alloc_trace.stack_trace.depth == 0) return;
+
   CpuThreadMatchingStatus status =
       CpuThreadMatchingStatus(alloc_trace.cpu_id == dealloc_trace.cpu_id,
                               alloc_trace.vcpu_id == dealloc_trace.vcpu_id,
@@ -630,6 +690,29 @@ void DeallocationProfiler::DeallocationStackTraceTable::AddTrace(
 
 void DeallocationProfiler::DeallocationStackTraceTable::Iterate(
     absl::FunctionRef<void(const Profile::Sample&)> func) const {
+  if (mode_ == Mode::kEventTrace) {
+    for (const auto& r : events_) {
+      tcmalloc::Profile::Sample s = {};
+      // Allocations have allocated_size > 0;
+      // Deallocations (matched and left-censored) have allocated_size == 0.
+      s.count = r.stack_trace.allocated_size > 0 ? 1 : -1;
+      s.allocation_time = r.stack_trace.allocation_time;
+      s.alloc_handle = r.stack_trace.sampled_alloc_handle;
+      s.allocated_size = r.stack_trace.allocated_size;
+      s.requested_size = r.stack_trace.requested_size;
+      s.cpu_id = r.cpu_id;
+      s.vcpu_id = r.vcpu_id;
+      s.l3_id = r.l3_id;
+      s.numa_id = r.numa_id;
+      s.thread_id = r.thread_id;
+      s.depth = std::min<size_t>(r.stack_trace.depth,
+                                 tcmalloc::Profile::Sample::kMaxStackDepth);
+      std::copy(r.stack_trace.stack, r.stack_trace.stack + s.depth, s.stack);
+      func(s);
+    }
+    return;
+  }
+
   uint64_t pair_id = 1;
 
   for (auto& it : table_) {
@@ -731,8 +814,9 @@ void DeallocationProfiler::DeallocationStackTraceTable::Iterate(
   }
 }
 
-DeallocationSample::DeallocationSample(DeallocationProfilerList* list) {
-  profiler_ = std::make_unique<DeallocationProfiler>(list);
+DeallocationSample::DeallocationSample(
+    DeallocationProfilerList* absl_nonnull list, Mode mode) {
+  profiler_ = std::make_unique<DeallocationProfiler>(list, mode);
 }
 
 DeallocationSample::~DeallocationSample() = default;

tcmalloc/deallocation_profiler.h

@@ -17,8 +17,8 @@
 
 #include <memory>
 
-#include "absl/base/const_init.h"
 #include "absl/base/internal/spinlock.h"
+#include "absl/base/nullability.h"
 #include "absl/time/time.h"
 #include "tcmalloc/internal/config.h"
 #include "tcmalloc/internal/logging.h"
@@ -46,10 +46,16 @@ class DeallocationProfilerList {
       absl::base_internal::SCHEDULE_KERNEL_ONLY};
 };
 
+// Lifetime profiling is essentially a time-aggregated view of an event trace,
+// so we share the majority of the implementation and switch internally in the
+// cases where the implementations must diverge.
+enum class Mode { kLifetimes, kEventTrace };
+
 class DeallocationSample final
     : public tcmalloc_internal::AllocationProfilingTokenBase {
  public:
-  explicit DeallocationSample(DeallocationProfilerList* absl_nonnull list);
+  explicit DeallocationSample(DeallocationProfilerList* absl_nonnull list,
+                              Mode mode);
   // We define the dtor to ensure it is placed in the desired text section.
   ~DeallocationSample() override;
 

tcmalloc/internal/parameter_accessors.h

@@ -88,6 +88,8 @@ TCMalloc_Internal_SetHugePageFillerSkipSubreleaseLongInterval(absl::Duration v);
 ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_GetMadviseColdRegionsNoHugepage();
 ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetMadviseColdRegionsNoHugepage(
     bool v);
+ABSL_ATTRIBUTE_WEAK int64_t TCMalloc_Internal_GetEventTraceMemoryLimit();
+ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetEventTraceMemoryLimit(int64_t v);
 ABSL_ATTRIBUTE_WEAK uint8_t TCMalloc_Internal_GetMinHotAccessHint();
 ABSL_ATTRIBUTE_WEAK void TCMalloc_Internal_SetMinHotAccessHint(uint8_t v);
 [[maybe_unused]] ABSL_ATTRIBUTE_WEAK bool TCMalloc_Internal_PossiblyCold(

tcmalloc/internal/profile_builder.cc

@@ -684,19 +684,33 @@ static absl::Status MakeLifetimeProfileProto(const tcmalloc::Profile& profile,
   // Common intern string ids which are going to be used for each sample.
   const int count_id = builder->InternString("count");
   const int nanoseconds_id = builder->InternString("nanoseconds");
-  const int avg_lifetime_id = builder->InternString("avg_lifetime");
-  const int stddev_lifetime_id = builder->InternString("stddev_lifetime");
-  const int min_lifetime_id = builder->InternString("min_lifetime");
-  const int max_lifetime_id = builder->InternString("max_lifetime");
+  const int cpu_raw_id = builder->InternString("cpu_id");
   const int active_cpu_id = builder->InternString("active CPU");
+  const int vcpu_raw_id = builder->InternString("vcpu_id");
   const int active_vcpu_id = builder->InternString("active vCPU");
+  const int l3_raw_id = builder->InternString("l3_id");
   const int active_l3_id = builder->InternString("active L3");
+  const int numa_raw_id = builder->InternString("numa_id");
   const int active_numa_id = builder->InternString("active NUMA");
+  const int thread_raw_id = builder->InternString("thread_id");
+  const int active_thread_id = builder->InternString("active thread");
+
+  // Lifetime profiling.
+  const int avg_lifetime_id = builder->InternString("avg_lifetime");
+  const int stddev_lifetime_id = builder->InternString("stddev_lifetime");
+  const int min_lifetime_id = builder->InternString("min_lifetime");
+  const int max_lifetime_id = builder->InternString("max_lifetime");
   const int same_id = builder->InternString("same");
   const int different_id = builder->InternString("different");
-  const int active_thread_id = builder->InternString("active thread");
-  const int callstack_pair_id = builder->InternString("callstack-pair-id");
   const int none_id = builder->InternString("none");
+  const int callstack_pair_id = builder->InternString("callstack-pair-id");
+
+  // Event tracing.
+  const int bytes_id = builder->InternString("bytes");
+  const int alloc_handle_id = builder->InternString("alloc_handle");
+  const int allocation_time_id = builder->InternString("allocation_time");
+  const int deallocation_time_id = builder->InternString("deallocation_time");
+  const int requested_size_id = builder->InternString("requested_size");
 
   profile.Iterate([&](const tcmalloc::Profile::Sample& entry) {
     perftools::profiles::Sample& sample = *converted.add_sample();
@@ -716,6 +730,13 @@ static absl::Status MakeLifetimeProfileProto(const tcmalloc::Profile& profile,
       add_label(key, unit, value);
     };
 
+    auto add_optional_int_label = [&](int key, int unit,
+                                      std::optional<int> opt_value) {
+      if (opt_value.has_value()) {
+        add_label(key, unit, static_cast<size_t>(opt_value.value()));
+      }
+    };
+
     auto add_optional_string_label =
         [&](int key, const std::optional<bool>& optional_result, int result1,
             int result2) {
@@ -744,25 +765,47 @@ static absl::Status MakeLifetimeProfileProto(const tcmalloc::Profile& profile,
     add_positive_label(max_lifetime_id, nanoseconds_id,
                        absl::ToInt64Nanoseconds(entry.max_lifetime));
 
+    add_optional_int_label(cpu_raw_id, 0, entry.cpu_id);
     add_optional_string_label(active_cpu_id,
                               entry.allocator_deallocator_physical_cpu_matched,
                               same_id, different_id);
+    add_optional_int_label(vcpu_raw_id, 0, entry.vcpu_id);
     add_optional_string_label(active_vcpu_id,
                               entry.allocator_deallocator_virtual_cpu_matched,
                               same_id, different_id);
+    add_optional_int_label(l3_raw_id, 0, entry.l3_id);
     add_optional_string_label(active_l3_id,
                               entry.allocator_deallocator_l3_matched, same_id,
                               different_id);
+    add_optional_int_label(numa_raw_id, 0, entry.numa_id);
     add_optional_string_label(active_numa_id,
                               entry.allocator_deallocator_numa_matched, same_id,
                               different_id);
+    add_optional_int_label(thread_raw_id, 0, entry.thread_id);
     add_optional_string_label(active_thread_id,
                               entry.allocator_deallocator_thread_matched,
                               same_id, different_id);
 
     int64_t count = abs(entry.count);
     int64_t weight = entry.sum;
 
+    if (auto handle = static_cast<uint64_t>(entry.alloc_handle); handle != 0) {
+      add_label(alloc_handle_id, count_id, handle);
+    }
+    // Set during event tracing, unset (epoch) during lifetime profiling.
+    if (entry.allocation_time > absl::UnixEpoch()) {
+      if (entry.count < 0) {  // Deallocation event
+        add_label(deallocation_time_id, nanoseconds_id,
+                  absl::ToUnixNanos(entry.allocation_time));
+      } else {  // Allocation event or censored allocation
+        add_label(allocation_time_id, nanoseconds_id,
+                  absl::ToUnixNanos(entry.allocation_time));
+      }
+    }
+    if (entry.count > 0) {  // Allocation event
+      add_label(requested_size_id, bytes_id, entry.requested_size);
+    }
+
     // Handle censored allocations first since we distinguish
     // the samples based on the is_censored flag.
     if (entry.is_censored) {
@@ -815,7 +858,8 @@ absl::StatusOr<std::unique_ptr<perftools::profiles::Profile>> MakeProfileProto(
   ProfileBuilder builder;
   builder.AddCurrentMappings();
 
-  if (profile.Type() == ProfileType::kLifetimes) {
+  if (profile.Type() == ProfileType::kLifetimes ||
+      profile.Type() == ProfileType::kEventTrace) {
     absl::Status error = MakeLifetimeProfileProto(profile, &builder);
     if (!error.ok()) {
       return error;

tcmalloc/internal_malloc_extension.h

@@ -73,6 +73,8 @@ ABSL_ATTRIBUTE_WEAK tcmalloc::tcmalloc_internal::AllocationProfilingTokenBase*
 MallocExtension_Internal_StartAllocationProfiling();
 ABSL_ATTRIBUTE_WEAK tcmalloc::tcmalloc_internal::AllocationProfilingTokenBase*
 MallocExtension_Internal_StartLifetimeProfiling();
+ABSL_ATTRIBUTE_WEAK tcmalloc::tcmalloc_internal::AllocationProfilingTokenBase*
+MallocExtension_Internal_StartEventTracing();
 
 ABSL_ATTRIBUTE_WEAK void MallocExtension_Internal_ActivateGuardedSampling();
 ABSL_ATTRIBUTE_WEAK tcmalloc::MallocExtension::Ownership

tcmalloc/malloc_extension.cc

@@ -305,6 +305,20 @@ MallocExtension::StartLifetimeProfiling() {
 #endif
 }
 
+MallocExtension::AllocationProfilingToken MallocExtension::StartEventTracing() {
+#if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS
+  if (&MallocExtension_Internal_StartEventTracing == nullptr) {
+    return {};
+  }
+
+  return tcmalloc_internal::AllocationProfilingTokenAccessor::MakeToken(
+      std::unique_ptr<tcmalloc_internal::AllocationProfilingTokenBase>(
+          MallocExtension_Internal_StartEventTracing()));
+#else
+  return {};
+#endif
+}
+
 void MallocExtension::MarkThreadIdle() {
 #if ABSL_INTERNAL_HAVE_WEAK_MALLOCEXTENSION_STUBS
   if (&MallocExtension_Internal_MarkThreadIdle == nullptr) {