Create a Perfetto Sink (#2169)

Summary:
Pull Request resolved: #2169

Create a TraceDispatcherSink that writes perfetto traces in their native protobuf format directly to local files

The advantages include:
- No row limits (vs 400k for a scuba query)
- Zero ingestion latency (Traces are available immediately)
- No json -> protobuf conversion needed and very compact
- Compatibility with distributed file systems like OILFS
- Faster execution of expanse: No more querying scuba and conversion of scuba -> perfetto protobuf

Differential Revision: D89403329

Author: thomasywang

hyperactor_telemetry/src/lib.rs

@@ -704,6 +704,27 @@ fn initialize_logging_with_log_prefix_impl(
                 }
             }
 
+            if hyperactor_config::global::get(sinks::perfetto::PERFETTO_TRACE_MODE)
+                != sinks::perfetto::PerfettoTraceMode::Off
+            {
+                let exec_id = env::execution_id();
+                let process_name = std::env::var("HYPERACTOR_PROCESS_NAME")
+                    .unwrap_or_else(|_| "client".to_string());
+                match sinks::perfetto::PerfettoFileSink::new(
+                    sinks::perfetto::default_trace_dir(),
+                    &exec_id,
+                    &process_name,
+                ) {
+                    Ok(sink) => {
+                        max_level = Some(tracing::level_filters::LevelFilter::TRACE);
+                        sinks.push(Box::new(sink));
+                    }
+                    Err(e) => {
+                        tracing::warn!("failed to create PerfettoFileSink: {}", e);
+                    }
+                }
+            }
+
             {
                 if hyperactor_config::global::get(ENABLE_OTEL_TRACING) {
                     use crate::meta;

hyperactor_telemetry/src/sinks/mod.rs

@@ -11,4 +11,5 @@
 //! writing events to a specific backend (SQLite, Scuba, glog, etc).
 
 pub mod glog;
+pub mod perfetto;
 pub mod sqlite;