fix(disk buffer): recover from decode errors during initialization seek

Apurva Nisal · Apurva Nisal · commit 02dc8b9b1b53 · 2026-06-29T02:47:23.000+05:30
Add ReaderError::Decode to is_bad_read() so seek_to_next_record during
buffer startup skips decode failures like checksum and partial_write errors,
instead of failing topology build with InvalidProtobufPayload.
diff --git a/changelog.d/9468_disk_buffer_decode_init_seek.fix.md b/changelog.d/9468_disk_buffer_decode_init_seek.fix.md
@@ -0,0 +1,3 @@
+Treat decode failures as recoverable bad reads during disk buffer initialization seek, preventing permanent startup failure when corrupted records are encountered during `seek_to_next_record`.
+
+authors: apurvanisal5
diff --git a/lib/vector-buffers/src/test/messages.rs b/lib/vector-buffers/src/test/messages.rs
@@ -231,6 +231,71 @@ impl FixedEncodable for UndecodableRecord {
     }
 }
 
+/// Like [`UndecodableRecord`], but the decode failure is controlled by the encoded flag.
+///
+/// `SelectiveDecodeRecord(true)` always fails to decode; `SelectiveDecodeRecord(false)` decodes
+/// successfully. Used to simulate corrupt middle records while keeping the last on-disk record
+/// valid for writer initialization.
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub(crate) struct SelectiveDecodeRecord(pub bool);
+
+impl AddBatchNotifier for SelectiveDecodeRecord {
+    fn add_batch_notifier(&mut self, batch: BatchNotifier) {
+        drop(batch);
+    }
+}
+
+impl ByteSizeOf for SelectiveDecodeRecord {
+    fn allocated_bytes(&self) -> usize {
+        0
+    }
+}
+
+impl EventCount for SelectiveDecodeRecord {
+    fn event_count(&self) -> usize {
+        1
+    }
+}
+
+impl FixedEncodable for SelectiveDecodeRecord {
+    type EncodeError = io::Error;
+    type DecodeError = io::Error;
+
+    fn encode<B>(self, buffer: &mut B) -> Result<(), Self::EncodeError>
+    where
+        B: BufMut,
+    {
+        if buffer.remaining_mut() < 1 {
+            return Err(io::Error::new(
+                io::ErrorKind::Other,
+                "not enough capacity to encode record",
+            ));
+        }
+
+        buffer.put_u8(u8::from(self.0));
+        Ok(())
+    }
+
+    fn decode<B>(mut buffer: B) -> Result<Self, Self::DecodeError>
+    where
+        B: Buf,
+    {
+        if buffer.remaining() < 1 {
+            return Err(io::Error::new(
+                io::ErrorKind::Other,
+                "not enough data to decode record",
+            ));
+        }
+
+        let fail_decode = buffer.get_u8() != 0;
+        if fail_decode {
+            return Err(io::Error::new(io::ErrorKind::Other, "failed to decode"));
+        }
+
+        Ok(SelectiveDecodeRecord(false))
+    }
+}
+
 message_wrapper!(MultiEventRecord: u32, |m: &Self| m.0);
 
 impl MultiEventRecord {
diff --git a/lib/vector-buffers/src/variants/disk_v2/reader.rs b/lib/vector-buffers/src/variants/disk_v2/reader.rs
@@ -134,6 +134,7 @@ where
             self,
             ReaderError::Checksum { .. }
                 | ReaderError::Deserialization { .. }
+                | ReaderError::Decode { .. }
                 | ReaderError::PartialWrite
         )
     }
diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/initialization.rs b/lib/vector-buffers/src/variants/disk_v2/tests/initialization.rs
@@ -4,7 +4,9 @@ use tokio::time::timeout;
 use tracing::Instrument;
 
 use crate::{
-    test::{SizedRecord, acknowledge, install_tracing_helpers, with_temp_dir},
+    test::{
+        acknowledge, install_tracing_helpers, with_temp_dir, SelectiveDecodeRecord, SizedRecord,
+    },
     variants::disk_v2::tests::{create_default_buffer_v2, set_file_length},
 };
 
@@ -182,3 +184,53 @@ async fn reader_doesnt_block_when_ahead_of_last_record_in_current_data_file() {
     let parent = trace_span!("reader_doesnt_block_when_ahead_of_last_record_in_current_data_file");
     fut.instrument(parent.or_current()).await;
 }
+
+#[tokio::test]
+async fn reader_recovers_from_decode_error_during_initialization_seek() {
+    // LOG-9468: if seek_to_next_record hits a decode failure (e.g. InvalidProtobufPayload),
+    // buffer open must not fail permanently — same recovery as partial_write/checksum.
+    let _a = install_tracing_helpers();
+
+    let fut = with_temp_dir(|dir| {
+        let data_dir = dir.to_path_buf();
+
+        async move {
+            let (mut writer, reader, ledger) = create_default_buffer_v2(data_dir.clone()).await;
+
+            // First record fails decode; second record decodes OK so writer startup validation passes.
+            writer
+                .write_record(SelectiveDecodeRecord(true))
+                .await
+                .expect("should not fail to write");
+            writer.flush().await.expect("flush should not fail");
+
+            writer
+                .write_record(SelectiveDecodeRecord(false))
+                .await
+                .expect("should not fail to write");
+            writer.flush().await.expect("flush should not fail");
+            writer.close();
+
+            // Simulate a restart where the reader had acknowledged through record ID 1.
+            unsafe { ledger.state().unsafe_set_reader_last_record_id(1) };
+            ledger.flush().expect("should not fail to flush ledger");
+
+            drop(reader);
+            drop(writer);
+            drop(ledger);
+
+            let reopen = timeout(
+                Duration::from_millis(500),
+                create_default_buffer_v2::<_, SelectiveDecodeRecord>(data_dir),
+            )
+            .await;
+            assert!(
+                reopen.is_ok(),
+                "buffer open should succeed after decode error during initialization seek"
+            );
+        }
+    });
+
+    let parent = trace_span!("reader_recovers_from_decode_error_during_initialization_seek");
+    fut.instrument(parent.or_current()).await;
+}

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	+Treat decode failures as recoverable bad reads during disk buffer initialization seek, preventing permanent startup failure when corrupted records are encountered during `seek_to_next_record`.
	`2`	`+`
	`3`	`+authors: apurvanisal5`
Original file line number	Diff line number	Diff line change
`@@ -134,6 +134,7 @@ where`
`134`	`134`	`self,`
`135`	`135`	`ReaderError::Checksum { .. }`
`136`	`136`	`\| ReaderError::Deserialization { .. }`
	`137`	`+ \| ReaderError::Decode { .. }`
`137`	`138`	`\| ReaderError::PartialWrite`
`138`	`139`	`)`
`139`	`140`	`}`