Skip to content

store reference timestamp in the event metadata #25698

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
gwenaskell wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

store reference timestamp in the event metadata #25698

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b03a9b6
store reference timestamp in the event metadata
gwenaskell Jun 29, 2026
25eb085
store the timestamp with the native format
gwenaskell Jun 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions lib/vector-core/src/event/metadata.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

use std::{borrow::Cow, collections::BTreeMap, fmt, sync::Arc, time::Instant};

use chrono::{DateTime, Utc};

use derivative::Derivative;
use lookup::OwnedTargetPath;
use serde::{Deserialize, Serialize};
Expand Down Expand Up @@ -87,6 +89,12 @@ pub(super) struct Inner {
/// An internal vector id that can be used to identify this event across all components.
#[derivative(PartialEq = "ignore")]
pub(crate) source_event_id: Option<Uuid>,

/// The wall-clock timestamp captured when the event batch first entered the source output
/// channel. Invariant across all fan-out copies of an event; set once at source send time
/// before any fan-out occurs.
#[serde(default, skip)]
pub(crate) reference_timestamp: Option<DateTime<Utc>>,
}

/// Metric Origin metadata for submission to Datadog.
Expand Down Expand Up @@ -262,6 +270,19 @@ impl EventMetadata {
pub fn set_last_transform_timestamp(&mut self, timestamp: Instant) {
self.last_transform_timestamp = Some(timestamp);
}

/// Returns the UTC timestamp captured when this event's batch entered the source output
/// channel, if available.
#[must_use]
pub fn reference_timestamp(&self) -> Option<DateTime<Utc>> {
self.inner.reference_timestamp
}

/// Sets the reference timestamp. Should be called once per event at source send time,
/// before fan-out. All fan-out copies will share this value via the shared `Arc<Inner>`.
pub fn set_reference_timestamp(&mut self, timestamp: DateTime<Utc>) {
self.get_mut().reference_timestamp = Some(timestamp);
}
}

impl Default for Inner {
Expand All @@ -277,6 +298,7 @@ impl Default for Inner {
dropped_fields: ObjectMap::new(),
datadog_origin_metadata: None,
source_event_id: Some(Uuid::new_v4()),
reference_timestamp: None,
}
}
}
Expand Down
15 changes: 9 additions & 6 deletions lib/vector-core/src/source_sender/output.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,15 +171,15 @@ impl Output {
events: EventArray,
unsent_event_count: &mut UnsentEventCount,
) -> Result<(), SendError> {
let reference = Utc::now().timestamp_millis();
let reference = Utc::now();
self.send_inner(events, unsent_event_count, reference).await
}

async fn send_inner(
&mut self,
mut events: EventArray,
unsent_event_count: &mut UnsentEventCount,
reference: i64,
reference: chrono::DateTime<Utc>,
) -> Result<(), SendError> {
// Apply post-processor with typed dispatch. Each method receives a reference to the
// concrete event type, making it impossible at the type level to change the variant.
Expand Down Expand Up @@ -209,16 +209,19 @@ impl Output {

// Emit lag time after the post-processor so that any timestamp mutations made by the
// processor are reflected in the metric.
let reference_millis = reference.timestamp_millis();
events
.iter_events()
.for_each(|event| self.emit_lag_time(event, reference));
.for_each(|event| self.emit_lag_time(event, reference_millis));

events.iter_events_mut().for_each(|mut event| {
let metadata = event.metadata_mut();
// attach runtime schema definitions from the source
if let Some(log_definition) = &self.log_definition {
event.metadata_mut().set_schema_definition(log_definition);
metadata.set_schema_definition(log_definition);
}
event.metadata_mut().set_upstream_id(Arc::clone(&self.id));
metadata.set_upstream_id(Arc::clone(&self.id));
metadata.set_reference_timestamp(reference);
});

let byte_size = events.estimated_json_encoded_size_of();
Expand Down Expand Up @@ -296,7 +299,7 @@ impl Output {
{
// Capture a single reference timestamp for the entire batch so that lag time
// measurements are not inflated by channel-send latency for later chunks.
let reference = Utc::now().timestamp_millis();
let reference = Utc::now();

// It's possible that the caller stops polling this future while it is blocked waiting
// on `self.send()`. When that happens, we use `UnsentEventCount` to correctly emit
Expand Down
Loading