Skip to content

Commit 7503f25

Browse files
JamesC1305JamesC1305
committed
fix: interrupts: replay legacy GSI allocations on restore
This is a follow-up from PR firecracker-microvm#5952, where we replayed MSI-X GSI allocations on snapshot restore, to ensure all of the handed out GSIs are valid at restore time. As part of that PR, we changed the `IdAllocator` struct to use a bitmap, backed by the `BitVec` struct from `bitvec` crate. `bitvec`'s implementation of serde deserialization doesn't handle malformed input gracefully, panicking instead of returning an error. Under normal conditions, where a well-formed snapshot is provided, such panics would never occur, but we would like to avoid a panic even under such conditions. To do this, we mirror the changes in firecracker-microvm#5952, creating a new `IdAllocator` on restore for the legacy GSIs, and replay allocations. This prevents us ever serializing bitvec and encountering this condition, with the added benefit of standardising behaviour across both GSI allocators. Signed-off-by: James Curtis <jxcurtis@amazon.co.uk>
1 parent a200d93 commit 7503f25

5 files changed

Lines changed: 39 additions & 12 deletions

File tree

‎src/vmm/src/device_manager/acpi.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ pub enum ACPIDeviceError {
1717
VmClock(#[from] VmClockError),
1818
/// Could not register IRQ with KVM: {0}
1919
RegisterIrq(#[from] kvm_ioctls::Error),
20+
/// Resource allocator error: {0}
21+
ResourceAllocator(#[from] vm_allocator::Error),
2022
}
2123

2224
// Although both VMGenID and VMClock devices are always present, they should be instantiated when
@@ -68,6 +70,17 @@ impl ACPIDeviceManager {
6870
Ok(())
6971
}
7072

73+
pub fn replay_gsi_allocations(&self, vm: &KvmVm) -> Result<(), ACPIDeviceError> {
74+
let mut resource_allocator = vm.resource_allocator();
75+
resource_allocator
76+
.gsi_legacy_allocator
77+
.allocate_id_at(self.vmgenid().gsi)?;
78+
resource_allocator
79+
.gsi_legacy_allocator
80+
.allocate_id_at(self.vmclock().gsi)?;
81+
Ok(())
82+
}
83+
7184
pub fn do_post_restore_vmgenid(&self) -> Result<(), ACPIDeviceError> {
7285
self.vmgenid().do_post_restore()?;
7386
Ok(())

‎src/vmm/src/device_manager/mmio.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,9 @@ impl MMIODeviceManager {
282282
// Create a new MMIODeviceInfo object on boot path or unwrap the
283283
// existing object on restore path.
284284
let device_info = if let Some(device_info) = device_info_opt {
285+
vm.resource_allocator()
286+
.gsi_legacy_allocator
287+
.allocate_id_at(device_info.gsi.ok_or(MmioError::InvalidIrqConfig)?)?;
285288
device_info
286289
} else {
287290
let gsi = vm.resource_allocator().allocate_gsi_legacy(1)?;
@@ -342,6 +345,9 @@ impl MMIODeviceManager {
342345
// Create a new MMIODeviceInfo object on boot path or unwrap the
343346
// existing object on restore path.
344347
let device_info = if let Some(device_info) = device_info_opt {
348+
vm.resource_allocator()
349+
.gsi_legacy_allocator
350+
.allocate_id_at(device_info.gsi.ok_or(MmioError::InvalidIrqConfig)?)?;
345351
device_info
346352
} else {
347353
let gsi = vm.resource_allocator().allocate_gsi_legacy(1)?;

‎src/vmm/src/device_manager/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -678,6 +678,8 @@ pub enum DevicePersistError {
678678
VirtioMem(#[from] VirtioMemPersistError),
679679
/// Could not activate device: {0}
680680
DeviceActivation(#[from] ActivateError),
681+
/// Resource allocator error: {0}
682+
ResourceAllocator(#[from] vm_allocator::Error),
681683
}
682684

683685
/// Errors for (de)serialization of the device manager.

‎src/vmm/src/device_manager/persist.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,8 @@ impl<'a> Persist<'a> for ACPIDeviceManager {
193193
VmClock::restore((), &state.vmclock).unwrap(),
194194
);
195195

196+
acpi_devices.replay_gsi_allocations(vm)?;
197+
196198
acpi_devices.activate_vmgenid(vm)?;
197199
acpi_devices.do_post_restore_vmgenid()?;
198200

@@ -402,6 +404,10 @@ impl<'a> Persist<'a> for MMIODeviceManager {
402404
.map_err(|()| DevicePersistError::MmioTransport)?,
403405
));
404406

407+
vm.resource_allocator()
408+
.gsi_legacy_allocator
409+
.allocate_id_at(device_info.gsi.ok_or(MmioError::InvalidIrqConfig)?)?;
410+
405411
dev_manager.register_mmio_virtio(
406412
vm,
407413
id.clone(),

‎src/vmm/src/vstate/resources.rs‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -110,10 +110,11 @@ impl ResourceAllocator {
110110
}
111111

112112
/// Serializable state for the resource allocator.
113+
///
114+
/// GSI allocators are reconstructed empty and repopulated from restored device state so malformed
115+
/// snapshots cannot provide allocator state that disagrees with the devices.
113116
#[derive(Debug, Clone, Serialize, Deserialize)]
114117
pub struct ResourceAllocatorState {
115-
/// Allocator for legacy device interrupt lines
116-
pub gsi_legacy_allocator: IdAllocator,
117118
/// Allocator for memory in the 32-bit MMIO address space
118119
pub mmio32_memory: AddressAllocator,
119120
/// Allocator for memory in the 64-bit MMIO address space
@@ -137,7 +138,6 @@ impl<'a> Persist<'a> for ResourceAllocator {
137138

138139
fn save(&self) -> Self::State {
139140
ResourceAllocatorState {
140-
gsi_legacy_allocator: self.gsi_legacy_allocator.clone(),
141141
mmio32_memory: self.mmio32_memory.clone(),
142142
mmio64_memory: self.mmio64_memory.clone(),
143143
past_mmio64_memory: self.past_mmio64_memory.clone(),
@@ -150,7 +150,7 @@ impl<'a> Persist<'a> for ResourceAllocator {
150150
state: &Self::State,
151151
) -> Result<Self, Self::Error> {
152152
Ok(ResourceAllocator {
153-
gsi_legacy_allocator: state.gsi_legacy_allocator.clone(),
153+
gsi_legacy_allocator: IdAllocator::new(arch::GSI_LEGACY_START, arch::GSI_LEGACY_END)?,
154154
gsi_msi_allocator: IdAllocator::new(arch::GSI_MSI_START, arch::GSI_MSI_END)?,
155155
mmio32_memory: state.mmio32_memory.clone(),
156156
mmio64_memory: state.mmio64_memory.clone(),
@@ -161,7 +161,7 @@ impl<'a> Persist<'a> for ResourceAllocator {
161161
}
162162

163163
/// An unique ID allocator that allows management of IDs in a given interval.
164-
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
164+
#[derive(Debug, Clone, PartialEq, Eq)]
165165
pub struct IdAllocator {
166166
// Beginning of the range of IDs that we want to manage.
167167
range_base: u32,
@@ -356,7 +356,7 @@ mod tests {
356356
}
357357

358358
#[test]
359-
fn test_persist_omits_msi_gsi_allocator() {
359+
fn test_persist_omits_gsi_allocators() {
360360
let mut allocator = ResourceAllocator::new();
361361

362362
let legacy_gsi = allocator.allocate_gsi_legacy(1).unwrap()[0];
@@ -369,16 +369,16 @@ mod tests {
369369
let state = allocator.save();
370370
let mut restored = ResourceAllocator::restore((), &state).unwrap();
371371

372-
// Legacy GSIs and MMIO ranges are serialized, so their allocations survive restore.
373-
assert_eq!(restored.allocate_gsi_legacy(1).unwrap()[0], legacy_gsi + 1);
372+
// GSI allocators are intentionally omitted from ResourceAllocatorState and replayed by
373+
// the restored devices, so they start empty after restore.
374+
assert_eq!(restored.allocate_gsi_legacy(1).unwrap()[0], legacy_gsi);
375+
assert_eq!(restored.allocate_gsi_msi(1).unwrap()[0], msi_gsi);
376+
377+
// Memory allocators are still serialized directly.
374378
restored
375379
.mmio32_memory
376380
.allocate(1024, 1024, AllocPolicy::ExactMatch(mmio_range.start()))
377381
.unwrap_err();
378-
379-
// MSI GSIs are intentionally omitted from ResourceAllocatorState and replayed by the
380-
// restored PCI devices, so the allocator starts empty after restore.
381-
assert_eq!(restored.allocate_gsi_msi(1).unwrap()[0], msi_gsi);
382382
}
383383
}
384384

0 commit comments

Comments
 (0)