e1000: fix I219 hang on reset

Clear the rings before reset to avoid a HW hang.

Inspired by em-7.7.8 and DPDK (1fc9701238edcf0541289b9ae15565b6d9d7ab30)

Reviewed by:	erj
MFC after:	2 weeks
Sponsored by:	BBOX.io
Pull Request:	https://github.com/freebsd/freebsd-src/pull/540

1 files changed, 117 insertions, 2 deletions

diff --git a/sys/dev/e1000/if_em.c b/sys/dev/e1000/if_em.c
index 7135f273fd94..6442b5f3f41e 100644
--- a/sys/dev/e1000/if_em.c
+++ b/sys/dev/e1000/if_em.c
@@ -351,6 +351,7 @@ static int	em_get_regs(SYSCTL_HANDLER_ARGS);
 
 static void	lem_smartspeed(struct e1000_softc *);
 static void	igb_configure_queues(struct e1000_softc *);
+static void	em_flush_desc_rings(struct e1000_softc *);
 
 
 /*********************************************************************
@@ -1913,6 +1914,10 @@ em_if_stop(if_ctx_t ctx)
 
 	INIT_DEBUGOUT("em_if_stop: begin");
 
+	/* I219 needs special flushing to avoid hangs */
+	if (sc->hw.mac.type >= e1000_pch_spt && sc->hw.mac.type < igb_mac_min)
+		em_flush_desc_rings(sc);
+
 	e1000_reset_hw(&sc->hw);
 	if (sc->hw.mac.type >= e1000_82544)
 		E1000_WRITE_REG(&sc->hw, E1000_WUFC, 0);
@@ -1978,8 +1983,7 @@ em_allocate_pci_resources(if_ctx_t ctx)
 	sc->hw.hw_addr = (u8 *)&sc->osdep.mem_bus_space_handle;
 
 	/* Only older adapters use IO mapping */
-	if (sc->hw.mac.type < em_mac_min &&
-	    sc->hw.mac.type > e1000_82543) {
+	if (sc->hw.mac.type < em_mac_min && sc->hw.mac.type > e1000_82543) {
 		/* Figure our where our IO BAR is ? */
 		for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
 			val = pci_read_config(dev, rid, 4);
@@ -2467,6 +2471,113 @@ igb_init_dmac(struct e1000_softc *sc, u32 pba)
 		E1000_WRITE_REG(hw, E1000_DMACR, 0);
 	}
 }
+/*********************************************************************
+ * The 3 following flush routines are used as a workaround in the
+ * I219 client parts and only for them.
+ *
+ * em_flush_tx_ring - remove all descriptors from the tx_ring
+ *
+ * We want to clear all pending descriptors from the TX ring.
+ * zeroing happens when the HW reads the regs. We assign the ring itself as
+ * the data of the next descriptor. We don't care about the data we are about
+ * to reset the HW.
+ **********************************************************************/
+static void
+em_flush_tx_ring(struct e1000_softc *sc)
+{
+	struct e1000_hw		*hw = &sc->hw;
+	struct tx_ring		*txr = &sc->tx_queues->txr;
+	struct e1000_tx_desc	*txd;
+	u32			tctl, txd_lower = E1000_TXD_CMD_IFCS;
+	u16			size = 512;
+
+	tctl = E1000_READ_REG(hw, E1000_TCTL);
+	E1000_WRITE_REG(hw, E1000_TCTL, tctl | E1000_TCTL_EN);
+
+	txd = &txr->tx_base[txr->tx_cidx_processed];
+
+	/* Just use the ring as a dummy buffer addr */
+	txd->buffer_addr = txr->tx_paddr;
+	txd->lower.data = htole32(txd_lower | size);
+	txd->upper.data = 0;
+
+	/* flush descriptors to memory before notifying the HW */
+	wmb();
+
+	E1000_WRITE_REG(hw, E1000_TDT(0), txr->tx_cidx_processed);
+	mb();
+	usec_delay(250);
+}
+
+/*********************************************************************
+ * em_flush_rx_ring - remove all descriptors from the rx_ring
+ *
+ * Mark all descriptors in the RX ring as consumed and disable the rx ring
+ **********************************************************************/
+static void
+em_flush_rx_ring(struct e1000_softc *sc)
+{
+	struct e1000_hw	*hw = &sc->hw;
+	u32		rctl, rxdctl;
+
+	rctl = E1000_READ_REG(hw, E1000_RCTL);
+	E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
+	E1000_WRITE_FLUSH(hw);
+	usec_delay(150);
+
+	rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0));
+	/* zero the lower 14 bits (prefetch and host thresholds) */
+	rxdctl &= 0xffffc000;
+	/*
+	 * update thresholds: prefetch threshold to 31, host threshold to 1
+	 * and make sure the granularity is "descriptors" and not "cache lines"
+	 */
+	rxdctl |= (0x1F | (1 << 8) | E1000_RXDCTL_THRESH_UNIT_DESC);
+	E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl);
+
+	/* momentarily enable the RX ring for the changes to take effect */
+	E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_EN);
+	E1000_WRITE_FLUSH(hw);
+	usec_delay(150);
+	E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
+}
+
+/*********************************************************************
+ * em_flush_desc_rings - remove all descriptors from the descriptor rings
+ *
+ * In I219, the descriptor rings must be emptied before resetting the HW
+ * or before changing the device state to D3 during runtime (runtime PM).
+ *
+ * Failure to do this will cause the HW to enter a unit hang state which can
+ * only be released by PCI reset on the device
+ *
+ **********************************************************************/
+static void
+em_flush_desc_rings(struct e1000_softc *sc)
+{
+	struct e1000_hw	*hw = &sc->hw;
+	device_t dev = sc->dev;
+	u16		hang_state;
+	u32		fext_nvm11, tdlen;
+
+	/* First, disable MULR fix in FEXTNVM11 */
+	fext_nvm11 = E1000_READ_REG(hw, E1000_FEXTNVM11);
+	fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX;
+	E1000_WRITE_REG(hw, E1000_FEXTNVM11, fext_nvm11);
+
+	/* do nothing if we're not in faulty state, or if the queue is empty */
+	tdlen = E1000_READ_REG(hw, E1000_TDLEN(0));
+	hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2);
+	if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen)
+		return;
+	em_flush_tx_ring(sc);
+
+	/* recheck, maybe the fault is caused by the rx ring */
+	hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2);
+	if (hang_state & FLUSH_DESC_REQUIRED)
+		em_flush_rx_ring(sc);
+}
+
 
 /*********************************************************************
  *
@@ -2698,6 +2809,10 @@ em_reset(if_ctx_t ctx)
 		break;
 	}
 
+	/* I219 needs some special flushing to avoid hangs */
+	if (sc->hw.mac.type >= e1000_pch_spt && sc->hw.mac.type < igb_mac_min)
+		em_flush_desc_rings(sc);
+
 	/* Issue a global reset */
 	e1000_reset_hw(hw);
 	if (hw->mac.type >= igb_mac_min) {