Wait for FW readiness before initializing command interface in mlx5core.

Before attempting to initialize the command interface we must wait till
the fw_initializing bit is clear.

If we fail to meet this condition the hardware will drop our
configuration, specifically the descriptors page address.  This scenario
can happen when the firmware is still executing an FLR flow and did not
finish yet so the driver needs to wait for that to finish.

Linux commits:
6c780a0267b8
b8a92577f4be.

MFC after:	3 days
Sponsored by:	Mellanox Technologies

2 files changed, 38 insertions, 8 deletions

diff --git a/sys/dev/mlx5/device.h b/sys/dev/mlx5/device.h
index 5e734985c881..f1e8c3f9e931 100644
--- a/sys/dev/mlx5/device.h
+++ b/sys/dev/mlx5/device.h
@@ -32,8 +32,10 @@
 #include <rdma/ib_verbs.h>
 #include <dev/mlx5/mlx5_ifc.h>
 
-#define FW_INIT_TIMEOUT_MILI 2000
-#define FW_INIT_WAIT_MS 2
+#define	FW_INIT_TIMEOUT_MILI		2000
+#define	FW_INIT_WAIT_MS			2
+#define	FW_PRE_INIT_TIMEOUT_MILI	120000
+#define	FW_INIT_WARN_MESSAGE_INTERVAL	20000
 
 #if defined(__LITTLE_ENDIAN)
 #define MLX5_SET_HOST_ENDIANNESS	0
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_main.c b/sys/dev/mlx5/mlx5_core/mlx5_main.c
index 27fe28802b20..b6a1d6843277 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_main.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_main.c
@@ -676,19 +676,33 @@ static inline int fw_initializing(struct mlx5_core_dev *dev)
 	return ioread32be(&dev->iseg->initializing) >> 31;
 }
 
-static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili)
+static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili,
+			u32 warn_time_mili)
 {
-	u64 end = jiffies + msecs_to_jiffies(max_wait_mili);
+	int warn = jiffies + msecs_to_jiffies(warn_time_mili);
+	int end = jiffies + msecs_to_jiffies(max_wait_mili);
 	int err = 0;
 
-	while (fw_initializing(dev)) {
+	MPASS(max_wait_mili > warn_time_mili);
+
+	while (fw_initializing(dev) == 1) {
 		if (time_after(jiffies, end)) {
 			err = -EBUSY;
 			break;
 		}
+		if (warn_time_mili && time_after(jiffies, warn)) {
+			mlx5_core_warn(dev,
+			    "Waiting for FW initialization, timeout abort in %lu s\n",
+			    jiffies_to_msecs(end - warn) / 1000);
+			warn = jiffies + msecs_to_jiffies(warn_time_mili);
+		}
 		msleep(FW_INIT_WAIT_MS);
 	}
 
+	if (err != 0)
+		mlx5_core_dbg(dev, "Full initializing bit dword = 0x%x\n",
+		    ioread32be(&dev->iseg->initializing));
+
 	return err;
 }
 
@@ -994,15 +1008,29 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
 	 */
 	dev->state = MLX5_DEVICE_STATE_UP;
 
+	/* wait for firmware to accept initialization segments configurations
+	*/
+	err = wait_fw_init(dev, FW_PRE_INIT_TIMEOUT_MILI,
+	    FW_INIT_WARN_MESSAGE_INTERVAL);
+	if (err) {
+		dev_err(&dev->pdev->dev,
+		    "Firmware over %d MS in pre-initializing state, aborting\n",
+		    FW_PRE_INIT_TIMEOUT_MILI);
+		goto out_err;
+	}
+
 	err = mlx5_cmd_init(dev);
 	if (err) {
-		mlx5_core_err(dev, "Failed initializing command interface, aborting\n");
+		mlx5_core_err(dev,
+		    "Failed initializing command interface, aborting\n");
 		goto out_err;
 	}
 
-	err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI);
+	err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI, 0);
 	if (err) {
-		mlx5_core_err(dev, "Firmware over %d MS in initializing state, aborting\n", FW_INIT_TIMEOUT_MILI);
+		mlx5_core_err(dev,
+		    "Firmware over %d MS in initializing state, aborting\n",
+		    FW_INIT_TIMEOUT_MILI);
 		goto err_cmd_cleanup;
 	}