Use PCI power-mgmt to reset a device if FLR fails.

A large number of devices don't support PCIe FLR, in particular
graphics adapters. Use PCI power management to perform the
reset if FLR fails or isn't available, by cycling the device
through the D3 state.

This has been tested by a number of users with Nvidia and AMD GPUs.

Submitted and tested by: Matt Macy
Reviewed by:	jhb, imp, rgrimes
MFC after:	3 weeks
Differential Revision:	https://reviews.freebsd.org/D15268

1 files changed, 26 insertions, 6 deletions

diff --git a/sys/amd64/vmm/io/ppt.c b/sys/amd64/vmm/io/ppt.c
index d7d3a29caa8b..a34b857ef35b 100644
--- a/sys/amd64/vmm/io/ppt.c
+++ b/sys/amd64/vmm/io/ppt.c
@@ -353,6 +353,30 @@ ppt_is_mmio(struct vm *vm, vm_paddr_t gpa)
 	return (FALSE);
 }
 
+static void
+ppt_pci_reset(device_t dev)
+{
+	int ps;
+
+	if (pcie_flr(dev,
+		     max(pcie_get_max_completion_timeout(dev) / 1000, 10),
+		     true))
+		return;
+
+	/*
+	 * If FLR fails, attempt a power-management reset by cycling
+	 * the device in/out of D3 state.
+	 * PCI spec says we can only go into D3 state from D0 state.
+	 * Transition from D[12] into D0 before going to D3 state.
+	 */
+	ps = pci_get_powerstate(dev);
+	if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
+		pci_set_powerstate(dev, PCI_POWERSTATE_D0);
+	if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3)
+		pci_set_powerstate(dev, PCI_POWERSTATE_D3);
+	pci_set_powerstate(dev, ps);
+}
+
 int
 ppt_assign_device(struct vm *vm, int bus, int slot, int func)
 {
@@ -368,9 +392,7 @@ ppt_assign_device(struct vm *vm, int bus, int slot, int func)
 			return (EBUSY);
 
 		pci_save_state(ppt->dev);
-		pcie_flr(ppt->dev,
-		    max(pcie_get_max_completion_timeout(ppt->dev) / 1000, 10),
-		    true);
+		ppt_pci_reset(ppt->dev);
 		pci_restore_state(ppt->dev);
 		ppt->vm = vm;
 		iommu_add_device(vm_iommu_domain(vm), pci_get_rid(ppt->dev));
@@ -393,9 +415,7 @@ ppt_unassign_device(struct vm *vm, int bus, int slot, int func)
 			return (EBUSY);
 
 		pci_save_state(ppt->dev);
-		pcie_flr(ppt->dev,
-		    max(pcie_get_max_completion_timeout(ppt->dev) / 1000, 10),
-		    true);
+		ppt_pci_reset(ppt->dev);
 		pci_restore_state(ppt->dev);
 		ppt_unmap_mmio(vm, ppt);
 		ppt_teardown_msi(ppt);