diff mbox series

[smartpqi,updates,V2,02/11] smartpqi: add controller handshake during kdump

Message ID 20210928235442.201875-3-don.brace@microchip.com (mailing list archive)
State Accepted
Headers show
Series smartpqi updates | expand

Commit Message

Don Brace Sept. 28, 2021, 11:54 p.m. UTC 
From: Mahesh Rajashekhara <mahesh.rajashekhara@microchip.com>

Correct kdump hangs when controller is locked up.

There are occasions when a controller reboot
(controller soft reset) is issued when a controller
firmware crash dump is in progress.

This leads to incomplete controller firmware crash dump.
 - When the controller crash dump is in progress,
   and a kdump is initiated, the driver issues
   inbound doorbell reset to bring back the
   controller in SIS mode.
 - If the controller is in locked up state,
   the inbound doorbell reset does not work causing
   controller initialization failures. This results
   in the driver hanging waiting for SIS mode.

To avoid an incomplete controller crash dump, add in
a controller crash dump handshake.
 - Controller will indicate start and end of the controller
   crash dump by setting some register bits.
 - Driver will look these bits when a kdump is initiated.
   If a controller crash dump is in progress, the driver will
   wait for the controller crash dump to complete
   before issuing the controller soft reset then complete
   driver initialization.

Reviewed-by: Scott Benesh <scott.benesh@microchip.com>
Reviewed-by: Scott Teel <scott.teel@microchip.com>
Reviewed-by: Mike McGowen <mike.mcgowen@microchip.com>
Signed-off-by: Mahesh Rajashekhara <mahesh.rajashekhara@microchip.com>
Signed-off-by: Don Brace <don.brace@microchip.com>
---
 drivers/scsi/smartpqi/smartpqi_init.c | 41 +++++++++++++++++++--
 drivers/scsi/smartpqi/smartpqi_sis.c  | 51 +++++++++++++++++++++++++++
 drivers/scsi/smartpqi/smartpqi_sis.h  |  1 +
 3 files changed, 91 insertions(+), 2 deletions(-)

Comments

John Donnelly Sept. 30, 2021, 6:21 p.m. UTC | #1 
On 9/28/21 6:54 PM, Don Brace wrote:
> From: Mahesh Rajashekhara <mahesh.rajashekhara@microchip.com>
> 
> Correct kdump hangs when controller is locked up.
> 
> There are occasions when a controller reboot
> (controller soft reset) is issued when a controller
> firmware crash dump is in progress.
> 
> This leads to incomplete controller firmware crash dump.
>   - When the controller crash dump is in progress,
>     and a kdump is initiated, the driver issues
>     inbound doorbell reset to bring back the
>     controller in SIS mode.
>   - If the controller is in locked up state,
>     the inbound doorbell reset does not work causing
>     controller initialization failures. This results
>     in the driver hanging waiting for SIS mode.
> 
> To avoid an incomplete controller crash dump, add in
> a controller crash dump handshake.
>   - Controller will indicate start and end of the controller
>     crash dump by setting some register bits.
>   - Driver will look these bits when a kdump is initiated.
>     If a controller crash dump is in progress, the driver will
>     wait for the controller crash dump to complete
>     before issuing the controller soft reset then complete
>     driver initialization.
> 
> Reviewed-by: Scott Benesh <scott.benesh@microchip.com>
> Reviewed-by: Scott Teel <scott.teel@microchip.com>
> Reviewed-by: Mike McGowen <mike.mcgowen@microchip.com>
> Signed-off-by: Mahesh Rajashekhara <mahesh.rajashekhara@microchip.com>
> Signed-off-by: Don Brace <don.brace@microchip.com>

Acked-by: John Donnelly <john.p.donnelly@oracle.com>


> ---
>   drivers/scsi/smartpqi/smartpqi_init.c | 41 +++++++++++++++++++--
>   drivers/scsi/smartpqi/smartpqi_sis.c  | 51 +++++++++++++++++++++++++++
>   drivers/scsi/smartpqi/smartpqi_sis.h  |  1 +
>   3 files changed, 91 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c
> index 97027574eb1f..5655d240f7a7 100644
> --- a/drivers/scsi/smartpqi/smartpqi_init.c
> +++ b/drivers/scsi/smartpqi/smartpqi_init.c
> @@ -234,15 +234,46 @@ static inline bool pqi_is_hba_lunid(u8 *scsi3addr)
>   	return pqi_scsi3addr_equal(scsi3addr, RAID_CTLR_LUNID);
>   }
>   
> +#define PQI_DRIVER_SCRATCH_PQI_MODE			0x1
> +#define PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED		0x2
> +
>   static inline enum pqi_ctrl_mode pqi_get_ctrl_mode(struct pqi_ctrl_info *ctrl_info)
>   {
> -	return sis_read_driver_scratch(ctrl_info);
> +	return sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_PQI_MODE ? PQI_MODE : SIS_MODE;
>   }
>   
>   static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,
>   	enum pqi_ctrl_mode mode)
>   {
> -	sis_write_driver_scratch(ctrl_info, mode);
> +	u32 driver_scratch;
> +
> +	driver_scratch = sis_read_driver_scratch(ctrl_info);
> +
> +	if (mode == PQI_MODE)
> +		driver_scratch |= PQI_DRIVER_SCRATCH_PQI_MODE;
> +	else
> +		driver_scratch &= ~PQI_DRIVER_SCRATCH_PQI_MODE;
> +
> +	sis_write_driver_scratch(ctrl_info, driver_scratch);
> +}
> +
> +static inline bool pqi_is_fw_triage_supported(struct pqi_ctrl_info *ctrl_info)
> +{
> +	return (sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED) != 0;
> +}
> +
> +static inline void pqi_save_fw_triage_setting(struct pqi_ctrl_info *ctrl_info, bool is_supported)
> +{
> +	u32 driver_scratch;
> +
> +	driver_scratch = sis_read_driver_scratch(ctrl_info);
> +
> +	if (is_supported)
> +		driver_scratch |= PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED;
> +	else
> +		driver_scratch &= ~PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED;
> +
> +	sis_write_driver_scratch(ctrl_info, driver_scratch);
>   }
>   
>   static inline void pqi_ctrl_block_scan(struct pqi_ctrl_info *ctrl_info)
> @@ -7292,6 +7323,7 @@ static void pqi_ctrl_update_feature_flags(struct pqi_ctrl_info *ctrl_info,
>   		ctrl_info->unique_wwid_in_report_phys_lun_supported =
>   			firmware_feature->enabled;
>   		break;
> +		pqi_save_fw_triage_setting(ctrl_info, firmware_feature->enabled);
>   	}
>   
>   	pqi_firmware_feature_status(ctrl_info, firmware_feature);
> @@ -7618,6 +7650,11 @@ static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)
>   	u32 product_id;
>   
>   	if (reset_devices) {
> +		if (pqi_is_fw_triage_supported(ctrl_info)) {
> +			rc = sis_wait_for_fw_triage_completion(ctrl_info);
> +			if (rc)
> +				return rc;
> +		}
>   		sis_soft_reset(ctrl_info);
>   		msleep(PQI_POST_RESET_DELAY_SECS * PQI_HZ);
>   	} else {
> diff --git a/drivers/scsi/smartpqi/smartpqi_sis.c b/drivers/scsi/smartpqi/smartpqi_sis.c
> index d63c46a8e38b..8acd3a80f582 100644
> --- a/drivers/scsi/smartpqi/smartpqi_sis.c
> +++ b/drivers/scsi/smartpqi/smartpqi_sis.c
> @@ -51,12 +51,20 @@
>   #define SIS_BASE_STRUCT_REVISION		9
>   #define SIS_BASE_STRUCT_ALIGNMENT		16
>   
> +#define SIS_CTRL_KERNEL_FW_TRIAGE		0x3
>   #define SIS_CTRL_KERNEL_UP			0x80
>   #define SIS_CTRL_KERNEL_PANIC			0x100
>   #define SIS_CTRL_READY_TIMEOUT_SECS		180
>   #define SIS_CTRL_READY_RESUME_TIMEOUT_SECS	90
>   #define SIS_CTRL_READY_POLL_INTERVAL_MSECS	10
>   
> +enum sis_fw_triage_status {
> +	FW_TRIAGE_NOT_STARTED = 0,
> +	FW_TRIAGE_STARTED,
> +	FW_TRIAGE_COND_INVALID,
> +	FW_TRIAGE_COMPLETED
> +};
> +
>   #pragma pack(1)
>   
>   /* for use with SIS_CMD_INIT_BASE_STRUCT_ADDRESS command */
> @@ -419,12 +427,55 @@ u32 sis_read_driver_scratch(struct pqi_ctrl_info *ctrl_info)
>   	return readl(&ctrl_info->registers->sis_driver_scratch);
>   }
>   
> +static inline enum sis_fw_triage_status
> +	sis_read_firmware_triage_status(struct pqi_ctrl_info *ctrl_info)
> +{
> +	return ((enum sis_fw_triage_status)(readl(&ctrl_info->registers->sis_firmware_status) &
> +		SIS_CTRL_KERNEL_FW_TRIAGE));
> +}
> +
>   void sis_soft_reset(struct pqi_ctrl_info *ctrl_info)
>   {
>   	writel(SIS_SOFT_RESET,
>   		&ctrl_info->registers->sis_host_to_ctrl_doorbell);
>   }
>   
> +#define SIS_FW_TRIAGE_STATUS_TIMEOUT_SECS		300
> +#define SIS_FW_TRIAGE_STATUS_POLL_INTERVAL_SECS		1
> +
> +int sis_wait_for_fw_triage_completion(struct pqi_ctrl_info *ctrl_info)
> +{
> +	int rc;
> +	enum sis_fw_triage_status status;
> +	unsigned long timeout;
> +
> +	timeout = (SIS_FW_TRIAGE_STATUS_TIMEOUT_SECS * PQI_HZ) + jiffies;
> +	while (1) {
> +		status = sis_read_firmware_triage_status(ctrl_info);
> +		if (status == FW_TRIAGE_COND_INVALID) {
> +			dev_err(&ctrl_info->pci_dev->dev,
> +				"firmware triage condition invalid\n");
> +			rc = -EINVAL;
> +			break;
> +		} else if (status == FW_TRIAGE_NOT_STARTED ||
> +			status == FW_TRIAGE_COMPLETED) {
> +			rc = 0;
> +			break;
> +		}
> +
> +		if (time_after(jiffies, timeout)) {
> +			dev_err(&ctrl_info->pci_dev->dev,
> +				"timed out waiting for firmware triage status\n");
> +			rc = -ETIMEDOUT;
> +			break;
> +		}
> +
> +		ssleep(SIS_FW_TRIAGE_STATUS_POLL_INTERVAL_SECS);
> +	}
> +
> +	return rc;
> +}
> +
>   static void __attribute__((unused)) verify_structures(void)
>   {
>   	BUILD_BUG_ON(offsetof(struct sis_base_struct,
> diff --git a/drivers/scsi/smartpqi/smartpqi_sis.h b/drivers/scsi/smartpqi/smartpqi_sis.h
> index d29c1352a826..c1db93054c86 100644
> --- a/drivers/scsi/smartpqi/smartpqi_sis.h
> +++ b/drivers/scsi/smartpqi/smartpqi_sis.h
> @@ -28,5 +28,6 @@ void sis_write_driver_scratch(struct pqi_ctrl_info *ctrl_info, u32 value);
>   u32 sis_read_driver_scratch(struct pqi_ctrl_info *ctrl_info);
>   void sis_soft_reset(struct pqi_ctrl_info *ctrl_info);
>   u32 sis_get_product_id(struct pqi_ctrl_info *ctrl_info);
> +int sis_wait_for_fw_triage_completion(struct pqi_ctrl_info *ctrl_info);
>   
>   #endif	/* _SMARTPQI_SIS_H */
>
diff mbox series

Patch

diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c
index 97027574eb1f..5655d240f7a7 100644
--- a/drivers/scsi/smartpqi/smartpqi_init.c
+++ b/drivers/scsi/smartpqi/smartpqi_init.c
@@ -234,15 +234,46 @@  static inline bool pqi_is_hba_lunid(u8 *scsi3addr)
 	return pqi_scsi3addr_equal(scsi3addr, RAID_CTLR_LUNID);
 }
 
+#define PQI_DRIVER_SCRATCH_PQI_MODE			0x1
+#define PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED		0x2
+
 static inline enum pqi_ctrl_mode pqi_get_ctrl_mode(struct pqi_ctrl_info *ctrl_info)
 {
-	return sis_read_driver_scratch(ctrl_info);
+	return sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_PQI_MODE ? PQI_MODE : SIS_MODE;
 }
 
 static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,
 	enum pqi_ctrl_mode mode)
 {
-	sis_write_driver_scratch(ctrl_info, mode);
+	u32 driver_scratch;
+
+	driver_scratch = sis_read_driver_scratch(ctrl_info);
+
+	if (mode == PQI_MODE)
+		driver_scratch |= PQI_DRIVER_SCRATCH_PQI_MODE;
+	else
+		driver_scratch &= ~PQI_DRIVER_SCRATCH_PQI_MODE;
+
+	sis_write_driver_scratch(ctrl_info, driver_scratch);
+}
+
+static inline bool pqi_is_fw_triage_supported(struct pqi_ctrl_info *ctrl_info)
+{
+	return (sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED) != 0;
+}
+
+static inline void pqi_save_fw_triage_setting(struct pqi_ctrl_info *ctrl_info, bool is_supported)
+{
+	u32 driver_scratch;
+
+	driver_scratch = sis_read_driver_scratch(ctrl_info);
+
+	if (is_supported)
+		driver_scratch |= PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED;
+	else
+		driver_scratch &= ~PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED;
+
+	sis_write_driver_scratch(ctrl_info, driver_scratch);
 }
 
 static inline void pqi_ctrl_block_scan(struct pqi_ctrl_info *ctrl_info)
@@ -7292,6 +7323,7 @@  static void pqi_ctrl_update_feature_flags(struct pqi_ctrl_info *ctrl_info,
 		ctrl_info->unique_wwid_in_report_phys_lun_supported =
 			firmware_feature->enabled;
 		break;
+		pqi_save_fw_triage_setting(ctrl_info, firmware_feature->enabled);
 	}
 
 	pqi_firmware_feature_status(ctrl_info, firmware_feature);
@@ -7618,6 +7650,11 @@  static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)
 	u32 product_id;
 
 	if (reset_devices) {
+		if (pqi_is_fw_triage_supported(ctrl_info)) {
+			rc = sis_wait_for_fw_triage_completion(ctrl_info);
+			if (rc)
+				return rc;
+		}
 		sis_soft_reset(ctrl_info);
 		msleep(PQI_POST_RESET_DELAY_SECS * PQI_HZ);
 	} else {
diff --git a/drivers/scsi/smartpqi/smartpqi_sis.c b/drivers/scsi/smartpqi/smartpqi_sis.c
index d63c46a8e38b..8acd3a80f582 100644
--- a/drivers/scsi/smartpqi/smartpqi_sis.c
+++ b/drivers/scsi/smartpqi/smartpqi_sis.c
@@ -51,12 +51,20 @@ 
 #define SIS_BASE_STRUCT_REVISION		9
 #define SIS_BASE_STRUCT_ALIGNMENT		16
 
+#define SIS_CTRL_KERNEL_FW_TRIAGE		0x3
 #define SIS_CTRL_KERNEL_UP			0x80
 #define SIS_CTRL_KERNEL_PANIC			0x100
 #define SIS_CTRL_READY_TIMEOUT_SECS		180
 #define SIS_CTRL_READY_RESUME_TIMEOUT_SECS	90
 #define SIS_CTRL_READY_POLL_INTERVAL_MSECS	10
 
+enum sis_fw_triage_status {
+	FW_TRIAGE_NOT_STARTED = 0,
+	FW_TRIAGE_STARTED,
+	FW_TRIAGE_COND_INVALID,
+	FW_TRIAGE_COMPLETED
+};
+
 #pragma pack(1)
 
 /* for use with SIS_CMD_INIT_BASE_STRUCT_ADDRESS command */
@@ -419,12 +427,55 @@  u32 sis_read_driver_scratch(struct pqi_ctrl_info *ctrl_info)
 	return readl(&ctrl_info->registers->sis_driver_scratch);
 }
 
+static inline enum sis_fw_triage_status
+	sis_read_firmware_triage_status(struct pqi_ctrl_info *ctrl_info)
+{
+	return ((enum sis_fw_triage_status)(readl(&ctrl_info->registers->sis_firmware_status) &
+		SIS_CTRL_KERNEL_FW_TRIAGE));
+}
+
 void sis_soft_reset(struct pqi_ctrl_info *ctrl_info)
 {
 	writel(SIS_SOFT_RESET,
 		&ctrl_info->registers->sis_host_to_ctrl_doorbell);
 }
 
+#define SIS_FW_TRIAGE_STATUS_TIMEOUT_SECS		300
+#define SIS_FW_TRIAGE_STATUS_POLL_INTERVAL_SECS		1
+
+int sis_wait_for_fw_triage_completion(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc;
+	enum sis_fw_triage_status status;
+	unsigned long timeout;
+
+	timeout = (SIS_FW_TRIAGE_STATUS_TIMEOUT_SECS * PQI_HZ) + jiffies;
+	while (1) {
+		status = sis_read_firmware_triage_status(ctrl_info);
+		if (status == FW_TRIAGE_COND_INVALID) {
+			dev_err(&ctrl_info->pci_dev->dev,
+				"firmware triage condition invalid\n");
+			rc = -EINVAL;
+			break;
+		} else if (status == FW_TRIAGE_NOT_STARTED ||
+			status == FW_TRIAGE_COMPLETED) {
+			rc = 0;
+			break;
+		}
+
+		if (time_after(jiffies, timeout)) {
+			dev_err(&ctrl_info->pci_dev->dev,
+				"timed out waiting for firmware triage status\n");
+			rc = -ETIMEDOUT;
+			break;
+		}
+
+		ssleep(SIS_FW_TRIAGE_STATUS_POLL_INTERVAL_SECS);
+	}
+
+	return rc;
+}
+
 static void __attribute__((unused)) verify_structures(void)
 {
 	BUILD_BUG_ON(offsetof(struct sis_base_struct,
diff --git a/drivers/scsi/smartpqi/smartpqi_sis.h b/drivers/scsi/smartpqi/smartpqi_sis.h
index d29c1352a826..c1db93054c86 100644
--- a/drivers/scsi/smartpqi/smartpqi_sis.h
+++ b/drivers/scsi/smartpqi/smartpqi_sis.h
@@ -28,5 +28,6 @@  void sis_write_driver_scratch(struct pqi_ctrl_info *ctrl_info, u32 value);
 u32 sis_read_driver_scratch(struct pqi_ctrl_info *ctrl_info);
 void sis_soft_reset(struct pqi_ctrl_info *ctrl_info);
 u32 sis_get_product_id(struct pqi_ctrl_info *ctrl_info);
+int sis_wait_for_fw_triage_completion(struct pqi_ctrl_info *ctrl_info);
 
 #endif	/* _SMARTPQI_SIS_H */