sfxge(4): add routine to check for hardware presence

Add efx_nic_hw_unavailable() routine to check for hardware presence
before continuing with NIC operations.

Submitted by:   Andy Moreton <amoreton at solarflare.com>
Sponsored by:   Solarflare Communications, Inc.
Differential Revision:  https://reviews.freebsd.org/D18260

7 files changed, 71 insertions, 4 deletions

diff --git a/sys/dev/sfxge/common/ef10_ev.c b/sys/dev/sfxge/common/ef10_ev.c
index 998e7e30a1a1..5cf5130d1fed 100644
--- a/sys/dev/sfxge/common/ef10_ev.c
+++ b/sys/dev/sfxge/common/ef10_ev.c
@@ -890,8 +890,9 @@ ef10_ev_rx(
 
 	EFX_EV_QSTAT_INCR(eep, EV_RX);
 
-	/* Discard events after RXQ/TXQ errors */
-	if (enp->en_reset_flags & (EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR))
+	/* Discard events after RXQ/TXQ errors, or hardware not available */
+	if (enp->en_reset_flags &
+	    (EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR | EFX_RESET_HW_UNAVAIL))
 		return (B_FALSE);
 
 	/* Basic packet information */
@@ -1091,8 +1092,9 @@ ef10_ev_tx(
 
 	EFX_EV_QSTAT_INCR(eep, EV_TX);
 
-	/* Discard events after RXQ/TXQ errors */
-	if (enp->en_reset_flags & (EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR))
+	/* Discard events after RXQ/TXQ errors, or hardware not available */
+	if (enp->en_reset_flags &
+	    (EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR | EFX_RESET_HW_UNAVAIL))
 		return (B_FALSE);
 
 	if (EFX_QWORD_FIELD(*eqp, ESF_DZ_TX_DROP_EVENT) != 0) {
diff --git a/sys/dev/sfxge/common/ef10_impl.h b/sys/dev/sfxge/common/ef10_impl.h
index d76f251bde33..cd7142483ea5 100644
--- a/sys/dev/sfxge/common/ef10_impl.h
+++ b/sys/dev/sfxge/common/ef10_impl.h
@@ -216,6 +216,10 @@ extern	__checkReturn	efx_rc_t
 ef10_nic_init(
 	__in		efx_nic_t *enp);
 
+extern	__checkReturn	boolean_t
+ef10_nic_hw_unavailable(
+	__in		efx_nic_t *enp);
+
 #if EFSYS_OPT_DIAG
 
 extern	__checkReturn	efx_rc_t
diff --git a/sys/dev/sfxge/common/ef10_nic.c b/sys/dev/sfxge/common/ef10_nic.c
index 8c5859323c2c..ae134a50901f 100644
--- a/sys/dev/sfxge/common/ef10_nic.c
+++ b/sys/dev/sfxge/common/ef10_nic.c
@@ -2331,6 +2331,28 @@ fail1:
 	return (rc);
 }
 
+	__checkReturn	boolean_t
+ef10_nic_hw_unavailable(
+	__in		efx_nic_t *enp)
+{
+	efx_dword_t dword;
+
+	if (enp->en_reset_flags & EFX_RESET_HW_UNAVAIL)
+		return (B_TRUE);
+
+	EFX_BAR_READD(enp, ER_DZ_BIU_MC_SFT_STATUS_REG, &dword, B_FALSE);
+	if (EFX_DWORD_FIELD(dword, EFX_DWORD_0) == 0xffffffff)
+		goto unavail;
+
+	return (B_FALSE);
+
+unavail:
+	EFSYS_PROBE(hw_unavail);
+	enp->en_reset_flags |= EFX_RESET_HW_UNAVAIL;
+
+	return (B_TRUE);
+}
+
 			void
 ef10_nic_fini(
 	__in		efx_nic_t *enp)
diff --git a/sys/dev/sfxge/common/efx.h b/sys/dev/sfxge/common/efx.h
index 997c8a930d0c..57189e944d68 100644
--- a/sys/dev/sfxge/common/efx.h
+++ b/sys/dev/sfxge/common/efx.h
@@ -183,6 +183,10 @@ extern	__checkReturn	efx_rc_t
 efx_nic_reset(
 	__in		efx_nic_t *enp);
 
+extern	__checkReturn	boolean_t
+efx_nic_hw_unavailable(
+	__in		efx_nic_t *enp);
+
 #if EFSYS_OPT_DIAG
 
 extern	__checkReturn	efx_rc_t
diff --git a/sys/dev/sfxge/common/efx_impl.h b/sys/dev/sfxge/common/efx_impl.h
index c52cc5fdd662..973914b218b4 100644
--- a/sys/dev/sfxge/common/efx_impl.h
+++ b/sys/dev/sfxge/common/efx_impl.h
@@ -87,6 +87,7 @@ extern "C" {
 #define	EFX_RESET_PHY		0x00000001
 #define	EFX_RESET_RXQ_ERR	0x00000002
 #define	EFX_RESET_TXQ_ERR	0x00000004
+#define	EFX_RESET_HW_UNAVAIL	0x00000008
 
 typedef enum efx_mac_type_e {
 	EFX_MAC_INVALID = 0,
@@ -384,6 +385,7 @@ typedef struct efx_nic_ops_s {
 	efx_rc_t	(*eno_get_vi_pool)(efx_nic_t *, uint32_t *);
 	efx_rc_t	(*eno_get_bar_region)(efx_nic_t *, efx_nic_region_t,
 					uint32_t *, size_t *);
+	boolean_t	(*eno_hw_unavailable)(efx_nic_t *);
 #if EFSYS_OPT_DIAG
 	efx_rc_t	(*eno_register_test)(efx_nic_t *);
 #endif	/* EFSYS_OPT_DIAG */
diff --git a/sys/dev/sfxge/common/efx_mcdi.c b/sys/dev/sfxge/common/efx_mcdi.c
index 467b3c67c271..ba9102c0050b 100644
--- a/sys/dev/sfxge/common/efx_mcdi.c
+++ b/sys/dev/sfxge/common/efx_mcdi.c
@@ -525,6 +525,12 @@ efx_mcdi_request_poll(
 	EFSYS_ASSERT(!emip->emi_ev_cpl);
 	emrp = emip->emi_pending_req;
 
+	/* Check if hardware is unavailable */
+	if (efx_nic_hw_unavailable(enp)) {
+		EFSYS_UNLOCK(enp->en_eslp, state);
+		return (B_FALSE);
+	}
+
 	/* Check for reboot atomically w.r.t efx_mcdi_request_start */
 	if (emip->emi_poll_cnt++ == 0) {
 		if ((rc = efx_mcdi_poll_reboot(enp)) != 0) {
diff --git a/sys/dev/sfxge/common/efx_nic.c b/sys/dev/sfxge/common/efx_nic.c
index 5fe5b19666e1..abb164a9214d 100644
--- a/sys/dev/sfxge/common/efx_nic.c
+++ b/sys/dev/sfxge/common/efx_nic.c
@@ -129,6 +129,7 @@ static const efx_nic_ops_t	__efx_nic_siena_ops = {
 	siena_nic_init,			/* eno_init */
 	NULL,				/* eno_get_vi_pool */
 	NULL,				/* eno_get_bar_region */
+	NULL,				/* eno_hw_unavailable */
 #if EFSYS_OPT_DIAG
 	siena_nic_register_test,	/* eno_register_test */
 #endif	/* EFSYS_OPT_DIAG */
@@ -148,6 +149,7 @@ static const efx_nic_ops_t	__efx_nic_hunt_ops = {
 	ef10_nic_init,			/* eno_init */
 	ef10_nic_get_vi_pool,		/* eno_get_vi_pool */
 	ef10_nic_get_bar_region,	/* eno_get_bar_region */
+	ef10_nic_hw_unavailable,	/* eno_hw_unavailable */
 #if EFSYS_OPT_DIAG
 	ef10_nic_register_test,		/* eno_register_test */
 #endif	/* EFSYS_OPT_DIAG */
@@ -167,6 +169,7 @@ static const efx_nic_ops_t	__efx_nic_medford_ops = {
 	ef10_nic_init,			/* eno_init */
 	ef10_nic_get_vi_pool,		/* eno_get_vi_pool */
 	ef10_nic_get_bar_region,	/* eno_get_bar_region */
+	ef10_nic_hw_unavailable,	/* eno_hw_unavailable */
 #if EFSYS_OPT_DIAG
 	ef10_nic_register_test,		/* eno_register_test */
 #endif	/* EFSYS_OPT_DIAG */
@@ -186,6 +189,7 @@ static const efx_nic_ops_t	__efx_nic_medford2_ops = {
 	ef10_nic_init,			/* eno_init */
 	ef10_nic_get_vi_pool,		/* eno_get_vi_pool */
 	ef10_nic_get_bar_region,	/* eno_get_bar_region */
+	ef10_nic_hw_unavailable,	/* eno_hw_unavailable */
 #if EFSYS_OPT_DIAG
 	ef10_nic_register_test,		/* eno_register_test */
 #endif	/* EFSYS_OPT_DIAG */
@@ -681,6 +685,29 @@ fail1:
 	return (rc);
 }
 
+	__checkReturn	boolean_t
+efx_nic_hw_unavailable(
+	__in		efx_nic_t *enp)
+{
+	const efx_nic_ops_t *enop = enp->en_enop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	/* NOTE: can be used by MCDI before NIC probe */
+
+	if (enop->eno_hw_unavailable != NULL) {
+		if ((enop->eno_hw_unavailable)(enp) != B_FALSE)
+			goto unavail;
+	}
+
+	return (B_FALSE);
+
+unavail:
+	EFSYS_PROBE(hw_unavail);
+
+	return (B_TRUE);
+}
+
+
 #if EFSYS_OPT_DIAG
 
 	__checkReturn	efx_rc_t