Newly added features and bug fixes in latest Microchip SmartPQI driver

It includes:

1)Newly added TMF feature.
2)Added newly Huawei & Inspur PCI ID's
3)Fixed smartpqi driver hangs in Z-Pool while running on FreeBSD12.1
4)Fixed flooding dmesg in kernel while the controller is offline during in ioctls.
5)Avoided unnecessary host memory allocation for rcb sg buffers.
6)Fixed race conditions while accessing internal rcb structure.
7)Fixed where Logical volumes exposing two different names to the OS it's due to the system memory is overwritten with DMA stale data.
8)Fixed dynamically unloading a smartpqi driver.
9)Added device_shutdown callback instead of deprecated shutdown_final kernel event in smartpqi driver.
10)Fixed where Os is crashed during physical drive hot removal during heavy IO.
11)Fixed OS crash during controller lockup/offline during heavy IO.
12)Fixed coverity issues in smartpqi driver
13)Fixed system crash while creating and deleting logical volume in a continuous loop.
14)Fixed where the volume size is not exposing to OS when it expands.
15)Added HC3 pci id's.

Reviewed by:		Scott Benesh (microsemi), Murthy Bhat (microsemi), imp
Differential Revision:	https://reviews.freebsd.org/D30182

Sponsored by:		Netflix

1 files changed, 350 insertions, 105 deletions

diff --git a/sys/dev/smartpqi/smartpqi_init.c b/sys/dev/smartpqi/smartpqi_init.c
index 398a3bc207d1..1f127cff21ec 100644
--- a/sys/dev/smartpqi/smartpqi_init.c
+++ b/sys/dev/smartpqi/smartpqi_init.c
@@ -1,6 +1,5 @@
 /*-
- * Copyright (c) 2018 Microsemi Corporation.
- * All rights reserved.
+ * Copyright 2016-2021 Microchip Technology, Inc. and/or its subsidiaries.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -28,13 +27,17 @@
 
 #include "smartpqi_includes.h"
 
+/* 5 mins timeout for quiesce */
+#define PQI_QUIESCE_TIMEOUT	300000
+
 /*
  * Request the adapter to get PQI capabilities supported.
  */
-static int pqisrc_report_pqi_capability(pqisrc_softstate_t *softs)
+static int
+pqisrc_report_pqi_capability(pqisrc_softstate_t *softs)
 {
 	int ret = PQI_STATUS_SUCCESS;
-
+	
 	DBG_FUNC("IN\n");
 
 	gen_adm_req_iu_t	admin_req;
@@ -79,7 +82,7 @@ static int pqisrc_report_pqi_capability(pqisrc_softstate_t *softs)
 	} else {
 		DBG_ERR("Failed to send admin req report pqi device capability\n");
 		goto err_admin_req;
-		
+
 	}
 
 	softs->pqi_dev_cap.max_iqs = capability->max_iqs;
@@ -108,6 +111,7 @@ static int pqisrc_report_pqi_capability(pqisrc_softstate_t *softs)
 	DBG_INIT("softs->ib_spanning_supported: %d\n", softs->ib_spanning_supported);
 	DBG_INIT("softs->ob_spanning_supported: %d\n", softs->ob_spanning_supported);
 
+
 	os_mem_free(softs, (void *)capability,
 		    REPORT_PQI_DEV_CAP_DATA_BUF_SIZE);
 	os_dma_mem_free(softs, &pqi_cap_dma_buf);
@@ -129,9 +133,9 @@ err_out:
 /*
  * Function used to deallocate the used rcb.
  */
-void pqisrc_free_rcb(pqisrc_softstate_t *softs, int req_count)
+void
+pqisrc_free_rcb(pqisrc_softstate_t *softs, int req_count)
 {
-
 	uint32_t num_req;
 	size_t size;
 	int i;
@@ -146,10 +150,12 @@ void pqisrc_free_rcb(pqisrc_softstate_t *softs, int req_count)
 	DBG_FUNC("OUT\n");
 }
 
+
 /*
  * Allocate memory for rcb and SG descriptors.
  */
-static int pqisrc_allocate_rcb(pqisrc_softstate_t *softs)
+static int
+pqisrc_allocate_rcb(pqisrc_softstate_t *softs)
 {
 	int ret = PQI_STATUS_SUCCESS;
 	int i = 0;
@@ -163,7 +169,7 @@ static int pqisrc_allocate_rcb(pqisrc_softstate_t *softs)
 	/* Set maximum outstanding requests */
 	/* The valid tag values are from 1, 2, ..., softs->max_outstanding_io
 	 * The rcb will be accessed by using the tag as index
-	 * * As 0 tag index is not used, we need to allocate one extra.
+	 * As 0 tag index is not used, we need to allocate one extra.
 	 */
 	softs->max_outstanding_io = softs->pqi_cap.max_outstanding_io;
 	num_req = softs->max_outstanding_io + 1;
@@ -217,15 +223,16 @@ err_out:
  * Function used to decide the operational queue configuration params
  * - no of ibq/obq, shared/non-shared interrupt resource, IU spanning support
  */
-void pqisrc_decide_opq_config(pqisrc_softstate_t *softs)
+void
+pqisrc_decide_opq_config(pqisrc_softstate_t *softs)
 {
 	uint16_t total_iq_elements;
 
 	DBG_FUNC("IN\n");
 
-	DBG_INIT("softs->intr_count : %d  softs->num_cpus_online : %d", 
+	DBG_INIT("softs->intr_count : %d  softs->num_cpus_online : %d",
 		softs->intr_count, softs->num_cpus_online);
-
+	
 	if (softs->intr_count == 1 || softs->num_cpus_online == 1) {
 		/* Share the event and Operational queue. */
 		softs->num_op_obq = 1;
@@ -233,17 +240,13 @@ void pqisrc_decide_opq_config(pqisrc_softstate_t *softs)
 	}
 	else {
 		/* Note :  One OBQ (OBQ0) reserved for event queue */
-		softs->num_op_obq = MIN(softs->num_cpus_online, 
+		softs->num_op_obq = MIN(softs->num_cpus_online,
 					softs->intr_count) - 1;
-		softs->num_op_obq = softs->intr_count - 1;
 		softs->share_opq_and_eventq = false;
 	}
-
-	/* 
-	 * softs->num_cpus_online is set as number of physical CPUs,
-	 * So we can have more queues/interrupts .
-	 */
-	if (softs->intr_count > 1)	 
+	/* If the available interrupt count is more than one,
+	we dont need to share the interrupt for IO and event queue */
+	if (softs->intr_count > 1)
 		softs->share_opq_and_eventq = false;
 
 	DBG_INIT("softs->num_op_obq : %d\n",softs->num_op_obq);
@@ -261,23 +264,23 @@ void pqisrc_decide_opq_config(pqisrc_softstate_t *softs)
 		softs->max_ib_iu_length =
 			(softs->max_ib_iu_length_per_fw / softs->ibq_elem_size) *
 			 softs->ibq_elem_size;
-					   
+
 	}
-	/* If Max. Outstanding IO came with Max. Spanning element count then, 
+	/* If Max. Outstanding IO came with Max. Spanning element count then,
 		needed elements per IO are multiplication of
 		Max.Outstanding IO and  Max.Spanning element */
-	total_iq_elements = (softs->max_outstanding_io * 
+	total_iq_elements = (softs->max_outstanding_io *
 		(softs->max_ib_iu_length / softs->ibq_elem_size));
 
 	softs->num_elem_per_op_ibq = total_iq_elements / softs->num_op_raid_ibq;
-	softs->num_elem_per_op_ibq = MIN(softs->num_elem_per_op_ibq, 
+	softs->num_elem_per_op_ibq = MIN(softs->num_elem_per_op_ibq,
 		softs->pqi_dev_cap.max_iq_elements);
 
-	softs->num_elem_per_op_obq = softs->max_outstanding_io / softs->num_op_obq; 
+	softs->num_elem_per_op_obq = softs->max_outstanding_io / softs->num_op_obq;
 	softs->num_elem_per_op_obq = MIN(softs->num_elem_per_op_obq,
 		softs->pqi_dev_cap.max_oq_elements);
 
-	softs->max_sg_per_iu = ((softs->max_ib_iu_length - 
+	softs->max_sg_per_iu = ((softs->max_ib_iu_length -
 				softs->ibq_elem_size) /
 				sizeof(sgt_t)) +
 				MAX_EMBEDDED_SG_IN_FIRST_IU;
@@ -293,11 +296,12 @@ void pqisrc_decide_opq_config(pqisrc_softstate_t *softs)
 /*
  * Configure the operational queue parameters.
  */
-int pqisrc_configure_op_queues(pqisrc_softstate_t *softs)
+int
+pqisrc_configure_op_queues(pqisrc_softstate_t *softs)
 {
 	int ret = PQI_STATUS_SUCCESS;
 
-	/* Get the PQI capability, 
+	/* Get the PQI capability,
 		REPORT PQI DEVICE CAPABILITY request */
 	ret = pqisrc_report_pqi_capability(softs);
 	if (ret) {
@@ -310,11 +314,11 @@ int pqisrc_configure_op_queues(pqisrc_softstate_t *softs)
 	softs->max_io_for_scsi_ml = softs->max_outstanding_io - PQI_RESERVED_IO_SLOTS_CNT;
 
 	/* Decide the Op queue configuration */
-	pqisrc_decide_opq_config(softs);	
+	pqisrc_decide_opq_config(softs);
 
 	DBG_FUNC("OUT\n");
 	return ret;
-		
+
 err_out:
 	DBG_FUNC("OUT failed\n");
 	return ret;
@@ -323,7 +327,8 @@ err_out:
 /*
  * Validate the PQI mode of adapter.
  */
-int pqisrc_check_pqimode(pqisrc_softstate_t *softs)
+int
+pqisrc_check_pqimode(pqisrc_softstate_t *softs)
 {
 	int ret = PQI_STATUS_FAILURE;
 	int tmo = 0;
@@ -335,7 +340,7 @@ int pqisrc_check_pqimode(pqisrc_softstate_t *softs)
 	tmo = PQISRC_PQIMODE_READY_TIMEOUT;
 	do {
 		signature = LE_64(PCI_MEM_GET64(softs, &softs->pqi_reg->signature, PQI_SIGNATURE));
-        
+
 		if (memcmp(&signature, PQISRC_PQI_DEVICE_SIGNATURE,
 				sizeof(uint64_t)) == 0) {
 			ret = PQI_STATUS_SUCCESS;
@@ -362,6 +367,7 @@ int pqisrc_check_pqimode(pqisrc_softstate_t *softs)
 		goto err_out;
 	}
 
+
 	tmo = PQISRC_PQIMODE_READY_TIMEOUT;
 	/* Check the PQI device status register */
 	COND_WAIT(LE_32(PCI_MEM_GET32(softs, &softs->pqi_reg->pqi_dev_status, PQI_DEV_STATUS)) &
@@ -379,11 +385,207 @@ err_out:
 	return ret;
 }
 
+/* PQI Feature processing */
+static int
+pqisrc_config_table_update(struct pqisrc_softstate *softs,
+	uint16_t first_section, uint16_t last_section)
+{
+	pqi_vendor_general_request_t request;
+	int ret = PQI_STATUS_FAILURE;
+
+	memset(&request, 0, sizeof(request));
+
+	request.header.iu_type = PQI_REQUEST_IU_VENDOR_GENERAL;
+	request.header.iu_length = sizeof(request) - PQI_REQUEST_HEADER_LENGTH;
+	request.function_code = PQI_VENDOR_GENERAL_CONFIG_TABLE_UPDATE;
+	request.data.config_table_update.first_section = first_section;
+	request.data.config_table_update.last_section = last_section;
+
+	ret = pqisrc_build_send_vendor_request(softs, &request, NULL);
+
+	if (ret != PQI_STATUS_SUCCESS) {
+		DBG_ERR("Failed to submit vendor general request IU, Ret status: %d\n", ret);
+		return PQI_STATUS_FAILURE;
+	}
+
+	return PQI_STATUS_SUCCESS;
+}
+
+static inline
+boolean_t pqi_is_firmware_feature_supported(
+	struct pqi_conf_table_firmware_features *firmware_feature_list,
+	unsigned int bit_position)
+{
+	unsigned int byte_index;
+
+	byte_index = bit_position / BITS_PER_BYTE;
+
+	if (byte_index >= firmware_feature_list->num_elements)
+		return false;
+
+	return firmware_feature_list->features_supported[byte_index] &
+		(1 << (bit_position % BITS_PER_BYTE)) ? true : false;
+}
+
+static inline
+boolean_t pqi_is_firmware_feature_enabled(
+	struct pqi_conf_table_firmware_features *firmware_feature_list,
+	uint8_t *firmware_features_addr, unsigned int bit_position)
+{
+	unsigned int byte_index;
+	uint8_t *feature_enabled_addr;
+
+	byte_index = (bit_position / BITS_PER_BYTE) +
+		(firmware_feature_list->num_elements * 2);
+
+	feature_enabled_addr = firmware_features_addr +
+		offsetof(struct pqi_conf_table_firmware_features,
+			features_supported) + byte_index;
+
+	return *feature_enabled_addr &
+		(1 << (bit_position % BITS_PER_BYTE)) ? true : false;
+}
+
+static inline void
+pqi_request_firmware_feature(
+	struct pqi_conf_table_firmware_features *firmware_feature_list,
+	unsigned int bit_position)
+{
+	unsigned int byte_index;
+
+	byte_index = (bit_position / BITS_PER_BYTE) +
+		firmware_feature_list->num_elements;
+
+	firmware_feature_list->features_supported[byte_index] |=
+		(1 << (bit_position % BITS_PER_BYTE));
+}
+
+/* Update PQI config table firmware features section and inform the firmware */
+static int
+pqisrc_set_host_requested_firmware_feature(pqisrc_softstate_t *softs,
+	struct pqi_conf_table_firmware_features *firmware_feature_list)
+{
+	uint8_t *request_feature_addr;
+	void *request_feature_abs_addr;
+
+	request_feature_addr = firmware_feature_list->features_supported +
+		firmware_feature_list->num_elements;
+	request_feature_abs_addr = softs->fw_features_section_abs_addr +
+		(request_feature_addr - (uint8_t*)firmware_feature_list);
+
+	os_io_memcpy(request_feature_abs_addr, request_feature_addr,
+			firmware_feature_list->num_elements);
+
+	return pqisrc_config_table_update(softs,
+		PQI_CONF_TABLE_SECTION_FIRMWARE_FEATURES,
+		PQI_CONF_TABLE_SECTION_FIRMWARE_FEATURES);
+}
+
+/* Check firmware has enabled the feature specified in the respective bit position. */
+inline boolean_t
+pqisrc_is_firmware_feature_enabled(pqisrc_softstate_t *softs,
+		struct pqi_conf_table_firmware_features *firmware_feature_list, uint16_t bit_position)
+{
+	uint16_t byte_index;
+	uint8_t *features_enabled_abs_addr;
+
+	byte_index = (bit_position / BITS_PER_BYTE) +
+		(firmware_feature_list->num_elements * 2);
+
+	features_enabled_abs_addr = softs->fw_features_section_abs_addr +
+	offsetof(struct pqi_conf_table_firmware_features,features_supported) + byte_index;
+
+	return *features_enabled_abs_addr &
+		(1 << (bit_position % BITS_PER_BYTE)) ? true : false;
+}
+
+static void
+pqi_firmware_feature_status(struct pqisrc_softstate	*softs,
+	struct pqi_firmware_feature *firmware_feature)
+{
+	switch(firmware_feature->feature_bit) {
+	case PQI_FIRMWARE_FEATURE_OFA:
+		break;
+	case PQI_FIRMWARE_FEATURE_TIMEOUT_IN_RAID_IU_SUPPORT:
+		softs->timeout_in_passthrough = true;
+		break;
+	case PQI_FIRMWARE_FEATURE_TIMEOUT_IN_TMF_IU_SUPPORT:
+		softs->timeout_in_tmf = true;
+		break;
+	default:
+		DBG_NOTE("Nothing to do \n");
+	}
+}
+
+/* Firmware features supported by the driver */
+static struct
+pqi_firmware_feature pqi_firmware_features[] = {
+	{
+		.feature_name = "Support timeout for pass-through commands",
+		.feature_bit = PQI_FIRMWARE_FEATURE_TIMEOUT_IN_RAID_IU_SUPPORT,
+		.feature_status = pqi_firmware_feature_status,
+	},
+	{
+		.feature_name = "Support timeout for LUN Reset TMF",
+		.feature_bit = PQI_FIRMWARE_FEATURE_TIMEOUT_IN_TMF_IU_SUPPORT,
+		.feature_status = pqi_firmware_feature_status,
+	}
+};
+
+static void
+pqisrc_process_firmware_features(pqisrc_softstate_t *softs)
+{
+	int rc;
+	struct pqi_conf_table_firmware_features *firmware_feature_list;
+	unsigned int i;
+	unsigned int num_features_requested;
+
+	firmware_feature_list = (struct pqi_conf_table_firmware_features*)
+		softs->fw_features_section_abs_addr;
+
+	/* Check features and request those supported by firmware and driver.*/
+	for (i = 0, num_features_requested = 0;
+		i < ARRAY_SIZE(pqi_firmware_features); i++) {
+		/* Firmware support it ? */
+		if (pqi_is_firmware_feature_supported(firmware_feature_list,
+				pqi_firmware_features[i].feature_bit)) {
+			pqi_request_firmware_feature(firmware_feature_list,
+				pqi_firmware_features[i].feature_bit);
+			pqi_firmware_features[i].supported = true;
+			num_features_requested++;
+			DBG_NOTE("%s supported by driver, requesting firmware to enable it\n",
+					pqi_firmware_features[i].feature_name);
+		} else {
+			DBG_NOTE("%s supported by driver, but not by current firmware\n",
+					pqi_firmware_features[i].feature_name);
+		}
+	}
+	if (num_features_requested == 0)
+		return;
+
+	rc = pqisrc_set_host_requested_firmware_feature(softs, firmware_feature_list);
+	if (rc) {
+		DBG_ERR("Failed to update pqi config table\n");
+		return;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(pqi_firmware_features); i++) {
+		if (pqi_is_firmware_feature_enabled(firmware_feature_list,
+			softs->fw_features_section_abs_addr, pqi_firmware_features[i].feature_bit)) {
+			pqi_firmware_features[i].enabled = true;
+			DBG_NOTE("Firmware feature %s enabled \n",pqi_firmware_features[i].feature_name);
+			if(pqi_firmware_features[i].feature_status)
+				pqi_firmware_features[i].feature_status(softs, &(pqi_firmware_features[i]));
+		}
+	}
+}
+
 /*
  * Get the PQI configuration table parameters.
  * Currently using for heart-beat counter scratch-pad register.
  */
-int pqisrc_process_config_table(pqisrc_softstate_t *softs)
+int
+pqisrc_process_config_table(pqisrc_softstate_t *softs)
 {
 	int ret = PQI_STATUS_FAILURE;
 	uint32_t config_table_size;
@@ -407,6 +609,13 @@ int pqisrc_process_config_table(pqisrc_softstate_t *softs)
 		return ret;
 	}
 
+	if (config_table_size < sizeof(conf_table) ||
+		config_table_size > PQI_CONF_TABLE_MAX_LEN) {
+		DBG_ERR("Invalid PQI conf table length of %u\n",
+			config_table_size);
+		goto out;
+	}
+
 	config_table_abs_addr = (uint8_t *)(softs->pci_mem_base_vaddr +
 					softs->pqi_cap.conf_tab_off);
 
@@ -414,6 +623,7 @@ int pqisrc_process_config_table(pqisrc_softstate_t *softs)
 			softs->pqi_cap.conf_tab_off,
 			(uint8_t*)conf_table, config_table_size);
 
+
 	if (memcmp(conf_table->sign, PQI_CONF_TABLE_SIGNATURE,
 			sizeof(conf_table->sign)) != 0) {
 		DBG_ERR("Invalid PQI config signature\n");
@@ -423,20 +633,23 @@ int pqisrc_process_config_table(pqisrc_softstate_t *softs)
 	section_off = LE_32(conf_table->first_section_off);
 
 	while (section_off) {
+
 		if (section_off+ sizeof(*section_hdr) >= config_table_size) {
-			DBG_ERR("PQI config table section offset (%u) beyond \
-			end of config table (config table length: %u)\n",
-					section_off, config_table_size);
+			DBG_INFO("Reached end of PQI config table. Breaking off.\n");
 			break;
 		}
-		
+
 		section_hdr = (struct pqi_conf_table_section_header *)((uint8_t *)conf_table + section_off);
-		
+
 		switch (LE_16(section_hdr->section_id)) {
 		case PQI_CONF_TABLE_SECTION_GENERAL_INFO:
-		case PQI_CONF_TABLE_SECTION_FIRMWARE_FEATURES:
 		case PQI_CONF_TABLE_SECTION_FIRMWARE_ERRATA:
 		case PQI_CONF_TABLE_SECTION_DEBUG:
+			break;
+		case PQI_CONF_TABLE_SECTION_FIRMWARE_FEATURES:
+			softs->fw_features_section_off = softs->pqi_cap.conf_tab_off + section_off;
+			softs->fw_features_section_abs_addr = softs->pci_mem_base_vaddr + softs->fw_features_section_off;
+			pqisrc_process_firmware_features(softs);
 		break;
 		case PQI_CONF_TABLE_SECTION_HEARTBEAT:
 		softs->heartbeat_counter_off = softs->pqi_cap.conf_tab_off +
@@ -460,7 +673,8 @@ out:
 }
 
 /* Wait for PQI reset completion for the adapter*/
-int pqisrc_wait_for_pqi_reset_completion(pqisrc_softstate_t *softs)
+int
+pqisrc_wait_for_pqi_reset_completion(pqisrc_softstate_t *softs)
 {
 	int ret = PQI_STATUS_SUCCESS;
 	pqi_reset_reg_t reset_reg;
@@ -476,7 +690,7 @@ int pqisrc_wait_for_pqi_reset_completion(pqisrc_softstate_t *softs)
 
 	while(1) {
 		if (pqi_reset_timeout++ == max_timeout) {
-			return PQI_STATUS_TIMEOUT; 
+			return PQI_STATUS_TIMEOUT;
 		}
 		OS_SLEEP(PQI_RESET_POLL_INTERVAL);/* 100 msec */
 		reset_reg.all_bits = PCI_MEM_GET32(softs,
@@ -491,7 +705,8 @@ int pqisrc_wait_for_pqi_reset_completion(pqisrc_softstate_t *softs)
 /*
  * Function used to perform PQI hard reset.
  */
-int pqi_reset(pqisrc_softstate_t *softs)
+int
+pqi_reset(pqisrc_softstate_t *softs)
 {
 	int ret = PQI_STATUS_SUCCESS;
 	uint32_t val = 0;
@@ -499,7 +714,7 @@ int pqi_reset(pqisrc_softstate_t *softs)
 
 	DBG_FUNC("IN\n");
 
-	if (true == softs->ctrl_in_pqi_mode) { 
+	if (true == softs->ctrl_in_pqi_mode) {
 
 		if (softs->pqi_reset_quiesce_allowed) {
 			val = PCI_MEM_GET32(softs, &softs->ioa_reg->host_to_ioa_db,
@@ -535,7 +750,8 @@ int pqi_reset(pqisrc_softstate_t *softs)
 /*
  * Initialize the adapter with supported PQI configuration.
  */
-int pqisrc_pqi_init(pqisrc_softstate_t *softs)
+int
+pqisrc_pqi_init(pqisrc_softstate_t *softs)
 {
 	int ret = PQI_STATUS_SUCCESS;
 
@@ -559,7 +775,7 @@ int pqisrc_pqi_init(pqisrc_softstate_t *softs)
 		goto err_out;
 	}
 
-	softs->intr_type = INTR_TYPE_NONE;	
+	softs->intr_type = INTR_TYPE_NONE;
 
 	/* Get the interrupt count, type, priority available from OS */
 	ret = os_get_intr_config(softs);
@@ -577,16 +793,16 @@ int pqisrc_pqi_init(pqisrc_softstate_t *softs)
 		sis_enable_intx(softs);
 	}
 
-	/* Create Admin Queue pair*/		
+	/* Create Admin Queue pair*/
 	ret = pqisrc_create_admin_queue(softs);
 	if(ret) {
                 DBG_ERR("Failed to configure admin queue\n");
                 goto err_admin_queue;
     	}
 
-	/* For creating event and IO operational queues we have to submit 
-	   admin IU requests.So Allocate resources for submitting IUs */  
-	     
+	/* For creating event and IO operational queues we have to submit
+	   admin IU requests.So Allocate resources for submitting IUs */
+
 	/* Allocate the request container block (rcb) */
 	ret = pqisrc_allocate_rcb(softs);
 	if (ret == PQI_STATUS_FAILURE) {
@@ -626,7 +842,7 @@ err_create_opq:
 err_config_opq:
 	pqisrc_destroy_taglist(softs,&softs->taglist);
 err_taglist:
-	pqisrc_free_rcb(softs, softs->max_outstanding_io + 1);		
+	pqisrc_free_rcb(softs, softs->max_outstanding_io + 1);
 err_rcb:
 	pqisrc_destroy_admin_queue(softs);
 err_admin_queue:
@@ -636,8 +852,8 @@ err_out:
 	return PQI_STATUS_FAILURE;
 }
 
-/* */
-int pqisrc_force_sis(pqisrc_softstate_t *softs)
+int
+pqisrc_force_sis(pqisrc_softstate_t *softs)
 {
 	int ret = PQI_STATUS_SUCCESS;
 
@@ -661,7 +877,7 @@ int pqisrc_force_sis(pqisrc_softstate_t *softs)
 	ret = pqi_reset(softs);
 	if (ret) {
 		return ret;
-	}	
+	}
 	/* Re enable SIS */
 	ret = pqisrc_reenable_sis(softs);
 	if (ret) {
@@ -670,24 +886,53 @@ int pqisrc_force_sis(pqisrc_softstate_t *softs)
 
 	PQI_SAVE_CTRL_MODE(softs, CTRL_SIS_MODE);
 
-	return ret;	
+	return ret;
 }
 
-int pqisrc_wait_for_cmnd_complete(pqisrc_softstate_t *softs)
+static int
+pqisrc_wait_for_cmnd_complete(pqisrc_softstate_t *softs)
 {
+	int count = 0;
 	int ret = PQI_STATUS_SUCCESS;
-	int tmo = PQI_CMND_COMPLETE_TMO;
 
-	COND_WAIT((softs->taglist.num_elem == softs->max_outstanding_io), tmo);
-	if (!tmo) {
-		DBG_ERR("Pending commands %x!!!",softs->taglist.num_elem);
-		ret = PQI_STATUS_TIMEOUT;
+	DBG_NOTE("softs->taglist.num_elem : %d",softs->taglist.num_elem);
+
+	if (softs->taglist.num_elem == softs->max_outstanding_io)
+		return ret;
+	else {
+		DBG_WARN("%d commands pending\n",
+		softs->max_outstanding_io - softs->taglist.num_elem);
+
+		while(1) {
+
+			/* Since heartbeat timer stopped ,check for firmware status*/
+			if (SIS_IS_KERNEL_PANIC(softs)) {
+				DBG_ERR("Controller FW is not running\n");
+				return PQI_STATUS_FAILURE;
+			}
+
+			if (softs->taglist.num_elem != softs->max_outstanding_io) {
+				/* Sleep for 1 msec */
+				OS_SLEEP(1000);
+				count++;
+				if(count % 1000 == 0) {
+					DBG_WARN("Waited for %d seconds", count/1000);
+				}
+				if (count >= PQI_QUIESCE_TIMEOUT) {
+					return PQI_STATUS_FAILURE;
+				}
+				continue;
+			}
+			break;
+		}
 	}
 	return ret;
 }
 
-void pqisrc_complete_internal_cmds(pqisrc_softstate_t *softs)
+static void
+pqisrc_complete_internal_cmds(pqisrc_softstate_t *softs)
 {
+
 	int tag = 0;
 	rcb_t *rcb;
 
@@ -700,10 +945,12 @@ void pqisrc_complete_internal_cmds(pqisrc_softstate_t *softs)
 	}
 }
 
+
 /*
  * Uninitialize the resources used during PQI initialization.
  */
-void pqisrc_pqi_uninit(pqisrc_softstate_t *softs)
+void
+pqisrc_pqi_uninit(pqisrc_softstate_t *softs)
 {
 	int i, ret;
 
@@ -715,29 +962,31 @@ void pqisrc_pqi_uninit(pqisrc_softstate_t *softs)
 	/* Wait for commands to complete */
 	ret = pqisrc_wait_for_cmnd_complete(softs);
 
+	/* disable and free the interrupt resources */
+	os_destroy_intr(softs);
+
 	/* Complete all pending commands. */
 	if(ret != PQI_STATUS_SUCCESS) {
 		pqisrc_complete_internal_cmds(softs);
 		os_complete_outstanding_cmds_nodevice(softs);
 	}
 
-    if(softs->devlist_lockcreated==true){    
-        os_uninit_spinlock(&softs->devlist_lock);
-        softs->devlist_lockcreated = false;
-    }
-    
+	if(softs->devlist_lockcreated==true){
+		os_uninit_spinlock(&softs->devlist_lock);
+		softs->devlist_lockcreated = false;
+	}
+
 	for (i = 0; i <  softs->num_op_raid_ibq; i++) {
-        /* OP RAID IB Q */
-        if(softs->op_raid_ib_q[i].lockcreated==true){
-		OS_UNINIT_PQILOCK(&softs->op_raid_ib_q[i].lock);
-		softs->op_raid_ib_q[i].lockcreated = false;
-        }
-        
-        /* OP AIO IB Q */
-        if(softs->op_aio_ib_q[i].lockcreated==true){
-		OS_UNINIT_PQILOCK(&softs->op_aio_ib_q[i].lock);
-		softs->op_aio_ib_q[i].lockcreated = false;
-        }
+		/* OP RAID IB Q */
+		if(softs->op_raid_ib_q[i].lockcreated==true){
+			OS_UNINIT_PQILOCK(&softs->op_raid_ib_q[i].lock);
+			softs->op_raid_ib_q[i].lockcreated = false;
+		}
+		/* OP AIO IB Q */
+		if(softs->op_aio_ib_q[i].lockcreated==true){
+			OS_UNINIT_PQILOCK(&softs->op_aio_ib_q[i].lock);
+			softs->op_aio_ib_q[i].lockcreated = false;
+		}
 	}
 
 	/* Free Op queues */
@@ -745,15 +994,17 @@ void pqisrc_pqi_uninit(pqisrc_softstate_t *softs)
 	os_dma_mem_free(softs, &softs->op_obq_dma_mem);
 	os_dma_mem_free(softs, &softs->event_q_dma_mem);
 
+
+
 	/* Free  rcb */
 	pqisrc_free_rcb(softs, softs->max_outstanding_io + 1);
 
 	/* Free request id lists */
 	pqisrc_destroy_taglist(softs,&softs->taglist);
 
-	if(softs->admin_ib_queue.lockcreated==true){
-		OS_UNINIT_PQILOCK(&softs->admin_ib_queue.lock);	
-        	softs->admin_ib_queue.lockcreated = false;
+	if(softs->admin_ib_queue.lockcreated==true) {
+		OS_UNINIT_PQILOCK(&softs->admin_ib_queue.lock);
+		softs->admin_ib_queue.lockcreated = false;
 	}
 
 	/* Free Admin Queue */
@@ -770,15 +1021,16 @@ void pqisrc_pqi_uninit(pqisrc_softstate_t *softs)
 /*
  * Function to initialize the adapter settings.
  */
-int pqisrc_init(pqisrc_softstate_t *softs)
+int
+pqisrc_init(pqisrc_softstate_t *softs)
 {
 	int ret = 0;
 	int i = 0, j = 0;
 
 	DBG_FUNC("IN\n");
-    
+
 	check_struct_sizes();
-    
+
 	/* Init the Sync interface */
 	ret = pqisrc_sis_init(softs);
 	if (ret) {
@@ -812,7 +1064,7 @@ int pqisrc_init(pqisrc_softstate_t *softs)
                 DBG_ERR(" Failed to configure Report events\n");
 		goto err_event;
 	}
-	 
+
 	/* Set event configuration*/
         ret = pqisrc_set_event_config(softs);
         if(ret){
@@ -834,7 +1086,7 @@ int pqisrc_init(pqisrc_softstate_t *softs)
 		goto err_host_wellness;
 	}
 
-    
+
 	os_strlcpy(softs->devlist_lock_name, "devlist_lock", LOCKNAME_SIZE);
 	ret = os_init_spinlock(softs, &softs->devlist_lock, softs->devlist_lock_name);
 	if(ret){
@@ -844,20 +1096,14 @@ int pqisrc_init(pqisrc_softstate_t *softs)
 	}
 	softs->devlist_lockcreated = true;
 
-	OS_ATOMIC64_SET(softs, num_intrs, 0);
-	softs->prev_num_intrs = softs->num_intrs;
-
 	/* Get the PQI configuration table to read heart-beat counter*/
-	if (PQI_NEW_HEARTBEAT_MECHANISM(softs)) {
-		ret = pqisrc_process_config_table(softs);
-		if (ret) {
-			DBG_ERR("Failed to process PQI configuration table %d\n", ret);
-			goto err_config_tab;
-		}
+	ret = pqisrc_process_config_table(softs);
+	if (ret) {
+		DBG_ERR("Failed to process PQI configuration table %d\n", ret);
+		goto err_config_tab;
 	}
 
-	if (PQI_NEW_HEARTBEAT_MECHANISM(softs))
-		softs->prev_heartbeat_count = CTRLR_HEARTBEAT_CNT(softs) - OS_FW_HEARTBEAT_TIMER_INTERVAL;
+	softs->prev_heartbeat_count = CTRLR_HEARTBEAT_CNT(softs) - OS_FW_HEARTBEAT_TIMER_INTERVAL;
 
 	/* Init device list */
 	for(i = 0; i < PQI_MAX_DEVICES; i++)
@@ -870,15 +1116,14 @@ int pqisrc_init(pqisrc_softstate_t *softs)
 	return ret;
 
 err_config_tab:
-	if(softs->devlist_lockcreated==true){    
+	if(softs->devlist_lockcreated==true){
 		os_uninit_spinlock(&softs->devlist_lock);
 		softs->devlist_lockcreated = false;
-	}	
+	}
 err_lock:
 err_fw_version:
 err_event:
 err_host_wellness:
-	os_destroy_intr(softs);
 err_intr:
 	pqisrc_pqi_uninit(softs);
 err_pqi:
@@ -894,7 +1139,8 @@ err_out:
  * Write all data in the adapter's battery-backed cache to
  * storage.
  */
-int pqisrc_flush_cache( pqisrc_softstate_t *softs,
+int
+pqisrc_flush_cache( pqisrc_softstate_t *softs,
 			enum pqisrc_flush_cache_event_type event_type)
 {
 	int rval = PQI_STATUS_SUCCESS;
@@ -906,7 +1152,7 @@ int pqisrc_flush_cache( pqisrc_softstate_t *softs,
 	if (pqisrc_ctrl_offline(softs))
 		return PQI_STATUS_FAILURE;
 
-	flush_buff = os_mem_alloc(softs, sizeof(pqisrc_bmic_flush_cache_t)); 
+	flush_buff = os_mem_alloc(softs, sizeof(pqisrc_bmic_flush_cache_t));
 	if (!flush_buff) {
 		DBG_ERR("Failed to allocate memory for flush cache params\n");
 		rval = PQI_STATUS_FAILURE;
@@ -936,7 +1182,8 @@ int pqisrc_flush_cache( pqisrc_softstate_t *softs,
 /*
  * Uninitialize the adapter.
  */
-void pqisrc_uninit(pqisrc_softstate_t *softs)
+void
+pqisrc_uninit(pqisrc_softstate_t *softs)
 {
 	DBG_FUNC("IN\n");
 
@@ -946,8 +1193,6 @@ void pqisrc_uninit(pqisrc_softstate_t *softs)
 
 	os_destroy_semaphore(&softs->scan_lock);
 
-	os_destroy_intr(softs);
-
 	pqisrc_cleanup_devices(softs);
 
 	DBG_FUNC("OUT\n");