Skip to content

Commit 3614c5a

Browse files
bltblt
committed
compress scratchbook
1 parent 2dae1f4 commit 3614c5a

50 files changed

Lines changed: 1490 additions & 11316 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎tests/antithesis/scenarios/vector_to_vector_e2e_disk/anytime_reload.sh‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,15 @@ set -euo pipefail
55
cfg="${VECTOR_CONFIG:?}"
66
alt="${VECTOR_CONFIG_ALT:?}"
77

8-
tmp="$(mktemp)"
9-
cp "$cfg" "$tmp"
10-
cp "$alt" "$cfg"
11-
cp "$tmp" "$alt"
12-
rm -f "$tmp"
8+
# Swap cfg and alt by writing each new file beside its target and renaming it into
9+
# place. rename(2) within a directory is atomic, so the fault profile terminating
10+
# this node mid-swap can only orphan a *.swap temp, never leave a half-written
11+
# config that breaks the next reload/restart. Both reads finish before either
12+
# rename, so the swap sees consistent originals.
13+
cp "$alt" "$cfg.swap"
14+
cp "$cfg" "$alt.swap"
15+
mv "$cfg.swap" "$cfg"
16+
mv "$alt.swap" "$alt"
1317

1418
# Vector is PID 1 in the node container. SIGHUP triggers reload-from-disk.
1519
kill -HUP 1

‎tests/antithesis/scenarios/vector_to_vector_e2e_disk/src/bin/eventually_conservation.rs‎

Lines changed: 19 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -219,25 +219,26 @@ async fn main() {
219219
&json!({ "delivered": report.delivered, "delivered_total": report.delivered_total })
220220
);
221221

222-
// New id each attempt, pass if any round-trips. A permanent wedge fails them all.
223-
if all_healthy(&client, &metrics_urls).await {
224-
let deadline = time::Instant::now() + time::Duration::from_secs(45);
225-
let mut progressed = false;
226-
while !progressed && time::Instant::now() < deadline {
227-
if let Some(probe) = claim(&client, &oracle_url).await {
228-
if post_probe(&client, &source_url, probe).await {
229-
time::sleep(time::Duration::from_secs(1)).await;
230-
progressed = delivered_contains(&client, &oracle_url, probe).await;
231-
}
232-
}
233-
if !progressed {
234-
time::sleep(time::Duration::from_secs(2)).await;
222+
// New id each attempt, pass if any round-trips. A permanent wedge — a node
223+
// that never recovers once faults stop — fails them all. The probe runs
224+
// unconditionally: an unhealthy node makes no progress and so fails here,
225+
// rather than skipping the check.
226+
let deadline = time::Instant::now() + time::Duration::from_secs(45);
227+
let mut progressed = false;
228+
while !progressed && time::Instant::now() < deadline {
229+
if let Some(probe) = claim(&client, &oracle_url).await {
230+
if post_probe(&client, &source_url, probe).await {
231+
time::sleep(time::Duration::from_secs(1)).await;
232+
progressed = delivered_contains(&client, &oracle_url, probe).await;
235233
}
236234
}
237-
assert_always!(
238-
progressed,
239-
"post-recovery write makes progress",
240-
&json!({ "acked": report.acked, "delivered": report.delivered })
241-
);
235+
if !progressed {
236+
time::sleep(time::Duration::from_secs(2)).await;
237+
}
242238
}
239+
assert_always!(
240+
progressed,
241+
"post-recovery write makes progress",
242+
&json!({ "acked": report.acked, "delivered": report.delivered })
243+
);
243244
}

‎tests/antithesis/scenarios/vector_to_vector_e2e_disk/src/bin/parallel_driver_produce.rs‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
extern crate antithesis_instrumentation;
1010

11-
use antithesis_sdk::{antithesis_init, assert_reachable};
11+
use antithesis_sdk::{antithesis_init, assert_reachable, assert_unreachable};
1212
use clap::Parser;
1313
use harness::payload_field;
1414
use serde_json::json;
@@ -52,14 +52,18 @@ async fn claim(client: &reqwest::Client, oracle_url: &str) -> Option<u64> {
5252
resp.text().await.ok()?.trim().parse().ok()
5353
}
5454

55-
/// Tell the oracle head acked this id, so it must come back.
56-
async fn report_acked(client: &reqwest::Client, oracle_url: &str, id: u64) {
57-
let _ = client
58-
.post(format!("{oracle_url}/acked"))
59-
.timeout(time::Duration::from_secs(10))
60-
.body(id.to_string())
61-
.send()
62-
.await;
55+
/// Tell the oracle head acked this id, so it must come back. Returns whether the
56+
/// oracle recorded the obligation.
57+
async fn report_acked(client: &reqwest::Client, oracle_url: &str, id: u64) -> bool {
58+
matches!(
59+
client
60+
.post(format!("{oracle_url}/acked"))
61+
.timeout(time::Duration::from_secs(10))
62+
.body(id.to_string())
63+
.send()
64+
.await,
65+
Ok(resp) if resp.status().is_success()
66+
)
6367
}
6468

6569
#[tokio::main(flavor = "current_thread")]
@@ -75,8 +79,20 @@ async fn main() {
7579
// Tight timeout. A head wedged by the underflow blocks forever, so we stop
7680
// waiting and retry the same id.
7781
if post_event(&client, &args.source_url, id, time::Duration::from_secs(5)).await {
78-
report_acked(&client, &args.oracle_url, id).await;
79-
assert_reachable!("produce driver got an end-to-end ack", &json!({ "id": id }));
82+
// head took durable responsibility, so the oracle must record the
83+
// obligation or a later loss of this id goes uncounted. /acked is a
84+
// loopback call to the oracle, which is never killed, frozen, or
85+
// network-faulted, so a failure here is anomalous: fail loudly rather
86+
// than leave an acked id the oracle never expects. The id is dropped;
87+
// the next invocation claims a fresh one.
88+
if report_acked(&client, &args.oracle_url, id).await {
89+
assert_reachable!("produce driver got an end-to-end ack", &json!({ "id": id }));
90+
} else {
91+
assert_unreachable!(
92+
"head acked an id but the oracle did not record the obligation",
93+
&json!({ "id": id })
94+
);
95+
}
8096
return;
8197
}
8298
time::sleep(time::Duration::from_millis(100)).await;

‎tests/antithesis/scratchbook/_external-references-digest.md‎

Lines changed: 46 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
# External References Digest (working note for discovery agents)
22

3-
This is scaffolding for the antithesis-research run on **disk buffers v2**
4-
(`lib/vector-buffers/src/variants/disk_v2/`). User scope answer: *"Whatever you
5-
have access to. You have your MCPs."* — so in-repo docs/RFCs plus Datadog
6-
internal doc/Jira were consulted. Key findings condensed below so per-focus agents
7-
don't need to re-fetch.
3+
Scaffolding for the antithesis-research run on **disk buffers v2**
4+
(`lib/vector-buffers/src/variants/disk_v2/`). User scope: *"Whatever you have
5+
access to. You have your MCPs."* — in-repo docs/RFCs plus Datadog internal doc/Jira.
6+
Findings condensed below so per-focus agents need not re-fetch.
87

98
## In-repo references
109

@@ -13,102 +12,68 @@ don't need to re-fetch.
1312
- `lib/vector-buffers/src/variants/disk_v2/mod.rs` — authoritative design doc
1413
(module-level comment): on-disk format, ledger, record IDs, recovery.
1514

16-
## Claimed guarantees (from `mod.rs` design doc + buffer spec + internal doc)
17-
18-
- Data files never exceed 128MB; ≤ 65,536 files; buffer ≤ ~8TB.
19-
- All records checksummed with **CRC32C**; records written
20-
sequentially/contiguously; a record never spans two data files.
21-
- Writers create+write data files; readers read+delete them. Reader deletes a
22-
data file **only after all records in it are acknowledged** (whole-file
23-
deletion, never partial truncation).
24-
- Ledger (`buffer.db`, memory-mapped) tracks `writer_next_record_id`,
25-
`writer_current_data_file_id`, `reader_current_data_file_id`,
26-
`reader_last_record_id`. Fields updated atomically, but **not** atomically
27-
w.r.t. reader/writer activity.
28-
- Record IDs are monotonic and encode event count: record ID N with next record
29-
M means the record holds M−N events. Used to compute buffer event count and to
30-
detect gaps / dropped events after corruption.
31-
- **Durability:** data is fsync'd every **500ms** (`DEFAULT_FLUSH_INTERVAL`).
32-
Page-cache flush happens on every `flush()` (readers see data immediately on
33-
Linux); full fsync only every 500ms. **Data-loss window on crash = up to 500ms
34-
of unsynced writes** (when e2e acks off). Graceful shutdown flushes everything
35-
→ no loss.
36-
- Min buffer `max_size` ~256MB; `DEFAULT_MAX_DATA_FILE_SIZE` 128MB;
37-
`DEFAULT_MAX_RECORD_SIZE` = 128MB; `DEFAULT_WRITE_BUFFER_SIZE` 256KB.
38-
- Endianness: files are host-endian; not portable across architectures.
39-
- Delivery semantics with e2e acks + disk buffer = **at-least-once**: crash after
40-
buffer write but before downstream ack → replay on restart → **possible
41-
duplicates** (downstream must dedup).
15+
Claimed guarantees (CRC32C, whole-file deletion, monotonic event-count IDs, 500ms
16+
fsync window, at-least-once with duplicates on replay, size constants) live in
17+
`sut-analysis.md` and the per-property files — not duplicated here.
4218

4319
## Known bugs / incidents (HIGH-VALUE Antithesis targets)
4420

4521
1. **Ledger `total_buffer_size` AtomicU64 underflow → permanent writer deadlock**
46-
(Vector #21683, partially mitigated by PR #23561 on the *reporter* side only;
47-
the ledger atomic still wraps).
48-
- `decrement_total_buffer_size` (ledger.rs, `fetch_sub` at ledger.rs:319) does
49-
raw `fetch_sub(amount, AcqRel)` with **no saturation**. If `amount >
50-
current_value`, the atomic wraps to ≈ 2^64. Under the `antithesis` feature
51-
this site now carries a committed `assert_always_greater_than_or_equal_to!(total_buffer_size, amount)`
52-
detector at ledger.rs:313 — it reports the wrap, it does not prevent the subtraction.
22+
(Vector #21683, mitigated by PR #23561 on the *reporter* side only; the ledger
23+
atomic still wraps).
24+
- `decrement_total_buffer_size` does raw `fetch_sub(amount, AcqRel)`, **no
25+
saturation**. If `amount > current_value` the atomic wraps to ≈ 2^64. Under the
26+
`antithesis` feature this site carries a committed
27+
`assert_always_greater_than_or_equal_to!(total_buffer_size, amount)` detector —
28+
reports the wrap, does not prevent the subtraction.
5329
- Then `total_buffer_size + unflushed_bytes` is always astronomical →
54-
`is_buffer_full()` returns true forever → `can_write_record()` false forever
55-
→ writer's `ensure_ready_for_write()` (writer.rs ~1001-1020) loops on
56-
`ledger.wait_for_reader().await` and never recovers. **Writer deadlocks
57-
permanently.**
58-
- Trigger: crash/reboot/abrupt-shutdown that leaves a data file whose on-disk
59-
size and readable-record bytes disagree, combined with the reader running
60-
through that file on restart. Partial writes at file-rotation boundaries are
61-
the most plausible cause. Not deterministic per-restart, but not exotic.
62-
- Reporter-side gauges use `saturating_sub` (PR #23561) so the *dashboard*
63-
no longer shows 2^64, but the ledger control-path atomic is unfixed.
30+
`is_buffer_full()` true forever → `can_write_record()` false forever → writer's
31+
`ensure_ready_for_write()` loops on `ledger.wait_for_reader().await` forever.
32+
**Writer deadlocks permanently.**
33+
- Trigger: crash/reboot/abrupt-shutdown leaving a data file whose on-disk size and
34+
readable-record bytes disagree, plus the reader running through it on restart.
35+
Partial writes at file-rotation boundaries are likeliest. Not deterministic
36+
per-restart, but not exotic.
37+
- Reporter gauges use `saturating_sub` (PR #23561) so the *dashboard* no longer
38+
shows 2^64, but the ledger control-path atomic is unfixed.
6439

65-
2. **Disk buffer stall + silent event drops during config reload**
66-
(Vector #24948, PR #24949; directly implicated in the **internal config-reload incident non-prod
67-
incident**).
40+
2. **Disk buffer stall + silent event drops during config reload** (Vector #24948,
41+
PR #24949; implicated in the **internal non-prod config-reload incident**).
6842
- Old writer dropped while events still in-flight → events lost without
6943
accounting.
7044
- `track_dropped_events` passes `0` for `byte_size` → permanent drift in
7145
buffer-size metrics.
72-
- `synchronize_buffer_usage()` re-seeds metrics while the old reporter may
73-
still run → double-counted metric spikes; then a metrics gap between old
74-
reporter teardown and the first tick (2s) of the new reporter.
46+
- `synchronize_buffer_usage()` re-seeds metrics while the old reporter may still
47+
run → double-counted spikes, then a metrics gap between old-reporter teardown
48+
and the new reporter's first tick (2s).
7549

7650
3. **`component_discarded_events_total` blind to buffer drops** (Vector #24606,
7751
#24144). When a disk buffer fills and `drop_newest` fires, only
78-
`buffer_discarded_events_total` increments; the component-level discarded
79-
counter stays 0 → silent data loss on dashboards. `BufferEventsDropped::emit()`
80-
in `lib/vector-buffers/src/internal_events.rs` never calls
81-
`ComponentEventsDropped`.
52+
`buffer_discarded_events_total` increments; the component-level discarded counter
53+
stays 0 → silent data loss on dashboards. `BufferEventsDropped::emit()` in
54+
`lib/vector-buffers/src/internal_events.rs` never calls `ComponentEventsDropped`.
8255

83-
4. **Buffer size gauges stuck non-zero / negative** (Vector #23995, #17666,
84-
#21683). Reporter `current() = total_entered.saturating_sub(total_left)`;
85-
stuck-at-non-zero still open.
56+
4. **Buffer size gauges stuck non-zero / negative** (Vector #23995, #17666, #21683).
57+
Reporter `current() = total_entered.saturating_sub(total_left)`; stuck-at-non-zero
58+
still open.
8659

87-
5. **Component tags lost for sinks using disk buffers** (OPA-5380): components
88-
paused for IO at init time lose `component_*` labels on later-registered
89-
metrics (utilization, etc.).
60+
5. **Component tags lost for sinks using disk buffers** (OPA-5380): components paused
61+
for IO at init time lose `component_*` labels on later-registered metrics
62+
(utilization, etc.).
9063

9164
## Existing test strategy (so we don't duplicate it)
9265

9366
- In-repo: extensive `proptest` + **model-based testing** under
94-
`variants/disk_v2/tests/model/` (a reference model + action sequencer +
95-
in-memory filesystem). Unit tests for acknowledgements, initialization,
96-
known_errors, size_limits, invariants, record.
97-
- Datadog internal: an E2E **chaos test** that SIGKILLs the worker 3× with e2e acks
98-
enabled and asserts every event is delivered end-to-end. Antithesis should go
99-
beyond: explore fault *timing/interleavings* (partial writes at rotation,
100-
fsync-vs-crash windows, reader/writer races on the mmap'd ledger) that a fixed
101-
3×SIGKILL test cannot.
67+
`variants/disk_v2/tests/model/` (reference model + action sequencer + in-memory
68+
filesystem). Unit tests for acknowledgements, initialization, known_errors,
69+
size_limits, invariants, record.
70+
- Datadog internal: an E2E **chaos test** SIGKILLing the worker 3× with e2e acks,
71+
asserting every event delivered end-to-end. Antithesis goes beyond, exploring
72+
fault *timing/interleavings* (partial writes at rotation, fsync-vs-crash windows,
73+
reader/writer races on the mmap'd ledger) a fixed 3×SIGKILL test cannot.
10274
- A **major lock-contention performance issue** affected all disk-buffer users
10375
(writer throughput ~90 MiB/s capped by contention) — points at writer/reader
10476
coordination hot paths.
10577

106-
## Notes on faults
107-
108-
- Crash-recovery properties require **node termination faults** (often disabled
109-
by default in Antithesis tenants) — flag this in the catalog.
110-
- The disk buffer is **single-process** (intra-Vector reader+writer sharing an
111-
mmap'd ledger). Network/partition faults are largely irrelevant to the buffer
112-
itself; the strong levers are node kill/restart, node hang, CPU throttling
113-
(exposes the fsync/flush timing windows and lock contention), and filesystem
114-
state across restart.
78+
(Fault-lever analysis — single-process buffer, node-kill required for
79+
crash-recovery — lives in `sut-analysis.md` and `deployment-topology.md`.)

0 commit comments

Comments
 (0)