@@ -160,3 +160,37 @@ config SCSI_UFS_BSG
Select this if you need a bsg device node for your UFS controller.
If unsure, say N.
+
+config SCSI_UFSHPB
+ bool "UFS Host Performance Booster (EXPERIMENTAL)"
+ depends on SCSI_UFSHCD
+ help
+ NAND flash-based storage devices, including UFS, have mechanisms
+ to translate logical addresses of IO requests to the corresponding
+ physical addresses of the flash storage. Traditionally this L2P
+ mapping data is loaded to the internal SRAM in the storage controller.
+ When the capacity of storage is larger, a larger size of SRAM for the
+ L2P map data is required. Since increased SRAM size affects the
+ manufacturing cost significantly, it is not cost-effective to allocate
+ all the amount of SRAM needed to keep all the Logical-address to
+ Physical-address (L2P) map data. Therefore, L2P map data, which is
+ required to identify the physical address for the requested IOs, can
+ only be partially stored in SRAM from NAND flash. Due to this partial
+ loading, accessing the flash address area where the L2P information
+ for that address is not loaded in the SRAM can result in serious
+ performance degradation.
+
+ UFS Host Performance Booster (HPB) is a software solution for the
+ above problem, which uses the host side system memory as a cache for
+ the FTL L2P mapping table. It does not need additional hardware
+ support from the host side. By using HPB, the L2P mapping table can be
+ read from host memory and stored in host-side memory. while reading
+ the operation, the corresponding L2P information will be sent to the
+ UFS device along with the reading request. Since the L2P entry is
+ provided in the read request, UFS device does not have to load L2P
+ entry from flash memory. This will significantly improve the read
+ performance.
+
+ Select this if you need HPB to random read performance improvement.
+ If unsure, say N.
+
@@ -7,6 +7,7 @@ obj-$(CONFIG_SCSI_UFS_QCOM) += ufs-qcom.o
obj-$(CONFIG_SCSI_UFSHCD) += ufshcd-core.o
ufshcd-core-y += ufshcd.o ufs-sysfs.o
ufshcd-core-$(CONFIG_SCSI_UFS_BSG) += ufs_bsg.o
+ufshcd-core-$(CONFIG_SCSI_UFSHPB) += ufshpb.o
obj-$(CONFIG_SCSI_UFSHCD_PCI) += ufshcd-pci.o
obj-$(CONFIG_SCSI_UFSHCD_PLATFORM) += ufshcd-pltfrm.o
obj-$(CONFIG_SCSI_UFS_HISI) += ufs-hisi.o
@@ -2341,6 +2341,12 @@ static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
lrbp->cmd->sc_data_direction);
ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
+
+#if defined(CONFIG_SCSI_UFSHPB)
+ /* If HPB works, retrieve L2P map entry from Host memory */
+ if (hba->ufshpb_state == HPB_PRESENT)
+ ufshpb_retrieve_l2p_entry(hba, lrbp);
+#endif
} else {
ret = -EINVAL;
}
@@ -4566,7 +4572,10 @@ static int ufshcd_slave_configure(struct scsi_device *sdev)
if (ufshcd_is_rpm_autosuspend_allowed(hba))
sdev->rpm_autosuspend = 1;
-
+#if defined(CONFIG_SCSI_UFSHPB)
+ if (sdev->lun < hba->dev_info.max_lu_supported)
+ hba->sdev_ufs_lu[sdev->lun] = sdev;
+#endif
return 0;
}
@@ -4682,6 +4691,16 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
*/
pm_runtime_get_noresume(hba->dev);
}
+
+#if defined(CONFIG_SCSI_UFSHPB)
+ /*
+ * HPB recommendations are provided in RESPONSE UPIU
+ * packets of successfully completed commands, which
+ * are commands terminated with GOOD status.
+ */
+ if (scsi_status == SAM_STAT_GOOD)
+ ufshpb_rsp_handler(hba, lrbp);
+#endif
break;
case UPIU_TRANSACTION_REJECT_UPIU:
/* TODO: handle Reject UPIU Response */
@@ -6025,6 +6044,11 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
}
}
spin_lock_irqsave(host->host_lock, flags);
+
+#if defined(CONFIG_SCSI_UFSHPB)
+ if (hba->ufshpb_state == HPB_PRESENT)
+ hba->ufshpb_state = HPB_NEED_RESET;
+#endif
ufshcd_transfer_req_compl(hba);
spin_unlock_irqrestore(host->host_lock, flags);
@@ -6032,6 +6056,11 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
hba->req_abort_count = 0;
ufshcd_update_reg_hist(&hba->ufs_stats.dev_reset, (u32)err);
if (!err) {
+#if defined(CONFIG_SCSI_UFSHPB)
+ if (hba->ufshpb_state == HPB_NEED_RESET)
+ schedule_delayed_work(&hba->ufshpb_init_work,
+ msecs_to_jiffies(10));
+#endif
err = SUCCESS;
} else {
dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
@@ -6559,16 +6588,16 @@ static int ufs_get_device_desc(struct ufs_hba *hba)
dev_info->ufs_features = desc_buf[DEVICE_DESC_PARAM_UFS_FEAT];
- if (desc_buf[DEVICE_DESC_PARAM_UFS_FEAT] & 0x80) {
- hba->dev_info.hpb_control_mode =
+ if (dev_info->ufs_features & 0x80) {
+ dev_info->hpb_control_mode =
desc_buf[DEVICE_DESC_PARAM_HPB_CTRL_MODE];
- hba->dev_info.hpb_ver =
+ dev_info->hpb_ver =
(u16) (desc_buf[DEVICE_DESC_PARAM_HPB_VER] << 8) |
desc_buf[DEVICE_DESC_PARAM_HPB_VER + 1];
dev_info(hba->dev, "HPB Version: 0x%2x\n",
- hba->dev_info.hpb_ver);
+ dev_info->hpb_ver);
dev_info(hba->dev, "HPB control mode: %d\n",
- hba->dev_info.hpb_control_mode);
+ dev_info->hpb_control_mode);
}
/*
* getting vendor (manufacturerID) and Bank Index in big endian
@@ -7113,6 +7142,14 @@ static void ufshcd_async_scan(void *data, async_cookie_t cookie)
/* Probe and add UFS logical units */
ret = ufshcd_add_lus(hba);
+
+#if defined(CONFIG_SCSI_UFSHPB)
+ /* Initialize HPB */
+ if (hba->ufshpb_state == HPB_NEED_INIT)
+ schedule_delayed_work(&hba->ufshpb_init_work,
+ msecs_to_jiffies(10));
+#endif
+
out:
/*
* If we failed to initialize the device or the device is not
@@ -8241,6 +8278,9 @@ EXPORT_SYMBOL(ufshcd_shutdown);
*/
void ufshcd_remove(struct ufs_hba *hba)
{
+#if defined(CONFIG_SCSI_UFSHPB)
+ ufshpb_release(hba, HPB_NEED_INIT);
+#endif
ufs_bsg_remove(hba);
ufs_sysfs_remove_nodes(hba->dev);
blk_cleanup_queue(hba->tmf_queue);
@@ -8512,6 +8552,9 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
*/
ufshcd_set_ufs_dev_active(hba);
+#if defined(CONFIG_SCSI_UFSHPB)
+ ufshpb_init(hba);
+#endif
async_schedule(ufshcd_async_scan, hba);
ufs_sysfs_add_nodes(hba->dev);
@@ -69,6 +69,7 @@
#include "ufs.h"
#include "ufshci.h"
+#include "ufshpb.h"
#define UFSHCD "ufshcd"
#define UFSHCD_DRIVER_VERSION "0.2"
@@ -711,6 +712,14 @@ struct ufs_hba {
struct device bsg_dev;
struct request_queue *bsg_queue;
+
+#if defined(CONFIG_SCSI_UFSHPB)
+ enum UFSHPB_STATE ufshpb_state;
+ struct ufshpb_lu *ufshpb_lup[32];
+ struct delayed_work ufshpb_init_work;
+ struct work_struct ufshpb_eh_work;
+ struct scsi_device *sdev_ufs_lu[32];
+#endif
};
/* Returns true if clocks can be gated. Otherwise false */
new file mode 100644
@@ -0,0 +1,3322 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Universal Flash Storage Host Performance Booster
+ *
+ * Copyright (C) 2017-2018 Samsung Electronics Co., Ltd.
+ *
+ * Authors:
+ * Yongmyung Lee <ymhungry.lee@samsung.com>
+ * Jinyoung Choi <j-young.choi@samsung.com>
+ */
+
+#include <linux/blkdev.h>
+#include <linux/blktrace_api.h>
+#include <linux/sysfs.h>
+#include <scsi/scsi.h>
+
+#include "ufs.h"
+#include "ufshcd.h"
+#include "ufshpb.h"
+
+#define hpb_trace(hpb, args...) \
+ do { if (hpb->hba->sdev_ufs_lu[hpb->lun] && \
+ hpb->hba->sdev_ufs_lu[hpb->lun]->request_queue) \
+ blk_add_trace_msg( \
+ hpb->hba->sdev_ufs_lu[hpb->lun]->request_queue, \
+ ##args); \
+ } while (0)
+
+/*
+ * debug variables
+ */
+int alloc_mctx;
+
+/*
+ * define global constants
+ */
+static int sects_per_blk_shift;
+static int bits_per_dword_shift;
+static int bits_per_dword_mask;
+static int bits_per_byte_shift;
+
+static int ufshpb_execute_cmd(struct ufshpb_lu *hpb, unsigned char *cmd);
+static int ufshpb_create_sysfs(struct ufs_hba *hba, struct ufshpb_lu *hpb);
+static int ufshpb_evict_one_region(struct ufshpb_lu *hpb, int *region);
+static inline int ufshpb_region_mctx_evict(struct ufshpb_lu *hpb,
+ struct ufshpb_region *cb);
+static int ufshpb_region_mctx_alloc(struct ufshpb_lu *hpb,
+ struct ufshpb_region *cb);
+static void ufshpb_subregion_dirty(struct ufshpb_lu *hpb,
+ struct ufshpb_subregion *cp);
+static inline void ufshpb_cleanup_lru_info(struct ufshpb_lu *hpb,
+ struct ufshpb_region *cb);
+static int ufshpb_l2p_mapping_load(struct ufshpb_lu *hpb, int region,
+ int subregion, struct ufshpb_map_ctx *mctx,
+ __u64 start_t, __u64 workq_enter_t);
+static void ufshpb_destroy_subregion_tbl(struct ufshpb_lu *hpb,
+ struct ufshpb_region *cb);
+
+static inline void ufshpb_get_bit_offset(struct ufshpb_lu *hpb,
+ int subregion_offset,
+ int *dword, int *offset)
+{
+ *dword = subregion_offset >> bits_per_dword_shift;
+ *offset = subregion_offset & bits_per_dword_mask;
+}
+
+static inline struct
+hpb_sense_data *ufshpb_get_sense_data(struct ufshcd_lrb *lrbp)
+{
+ return (struct hpb_sense_data *)&lrbp->ucd_rsp_ptr->sr.sense_data_len;
+}
+
+static inline void ufshpb_prepare_read_buf_cmd(unsigned char *cmd, int region,
+ int subregion, int alloc_len)
+{
+ cmd[0] = UFSHPB_READ_BUFFER;
+ cmd[1] = 0x01;
+ cmd[2] = GET_BYTE_1(region);
+ cmd[3] = GET_BYTE_0(region);
+ cmd[4] = GET_BYTE_1(subregion);
+ cmd[5] = GET_BYTE_0(subregion);
+ cmd[6] = GET_BYTE_2(alloc_len);
+ cmd[7] = GET_BYTE_1(alloc_len);
+ cmd[8] = GET_BYTE_0(alloc_len);
+ cmd[9] = 0x00;
+}
+
+static inline void ufshpb_prepare_write_buf_cmd(unsigned char *cmd, int region)
+{
+ cmd[0] = UFSHPB_WRITE_BUFFER;
+ cmd[1] = 0x01;
+ cmd[2] = GET_BYTE_1(region);
+ cmd[3] = GET_BYTE_0(region);
+ cmd[9] = 0x00;
+}
+
+static void ufshpb_prepare_ppn(struct ufshpb_lu *hpb, struct ufshcd_lrb *lrbp,
+ unsigned long long ppn, int blk_cnt)
+{
+ unsigned char cmd[16] = { 0 };
+ /*
+ * cmd[14] = Transfer length
+ */
+ cmd[0] = UFSHPB_READ;
+ cmd[2] = lrbp->cmd->cmnd[2];
+ cmd[3] = lrbp->cmd->cmnd[3];
+ cmd[4] = lrbp->cmd->cmnd[4];
+ cmd[5] = lrbp->cmd->cmnd[5];
+ cmd[6] = GET_BYTE_7(ppn);
+ cmd[7] = GET_BYTE_6(ppn);
+ cmd[8] = GET_BYTE_5(ppn);
+ cmd[9] = GET_BYTE_4(ppn);
+ cmd[10] = GET_BYTE_3(ppn);
+ cmd[11] = GET_BYTE_2(ppn);
+ cmd[12] = GET_BYTE_1(ppn);
+ cmd[13] = GET_BYTE_0(ppn);
+ cmd[14] = blk_cnt;
+ cmd[15] = 0x00;
+ memcpy(lrbp->cmd->cmnd, cmd, 16);
+ memcpy(lrbp->ucd_req_ptr->sc.cdb, cmd, 16);
+}
+
+static inline unsigned long long ufshpb_get_ppn(struct ufshpb_map_ctx *mctx,
+ int pos)
+{
+ unsigned long long *ppn_table;
+ struct page *page;
+ int index, offset;
+
+ index = pos / HPB_ENTREIS_PER_OS_PAGE;
+ offset = pos % HPB_ENTREIS_PER_OS_PAGE;
+
+ page = mctx->m_page[index];
+ BUG_ON(!page);
+
+ ppn_table = page_address(page);
+ BUG_ON(!ppn_table);
+
+ return ppn_table[offset];
+}
+
+/*
+ * Note: hpb->hpb_lock (irq) should be held before calling this function
+ */
+static inline void ufshpb_l2p_entry_dirty_bit_set(struct ufshpb_lu *hpb,
+ struct ufshpb_region *cb,
+ struct ufshpb_subregion *cp,
+ int dword, int offset,
+ unsigned int count)
+{
+ const unsigned long mask = ((1UL << count) - 1) & 0xffffffff;
+
+ if (cb->region_state == REGION_INACTIVE ||
+ cp->subregion_state == SUBREGION_DIRTY)
+ return;
+
+ BUG_ON(!cp->mctx);
+ cp->mctx->ppn_dirty[dword] |= (mask << offset);
+ cp->mctx->ppn_dirty_counter += count;
+
+ if (cp->mctx->ppn_dirty_counter >= hpb->entries_per_subregion)
+ ufshpb_subregion_dirty(hpb, cp);
+}
+
+/*
+ * Check if this entry/ppn is dirty or not in the L2P table cache,
+ * Note: hpb->hpb_lock (irq) should be held before calling this function
+ */
+static bool ufshpb_l2p_entry_dirty_check(struct ufshpb_lu *hpb,
+ struct ufshpb_subregion *cp,
+ int subregion_offset)
+{
+ bool is_dirty;
+ unsigned int bit_dword, bit_offset;
+
+ if (!cp->mctx->ppn_dirty || cp->subregion_state == SUBREGION_DIRTY)
+ return false;
+
+ ufshpb_get_bit_offset(hpb, subregion_offset,
+ &bit_dword, &bit_offset);
+
+ is_dirty = cp->mctx->ppn_dirty[bit_dword] &
+ (1 << bit_offset) ? true : false;
+
+ return is_dirty;
+}
+
+static void ufshpb_l2p_entry_dirty_set(struct ufshpb_lu *hpb,
+ struct ufshcd_lrb *lrbp,
+ int region, int subregion,
+ int subregion_offset)
+{
+ struct ufshpb_region *cb;
+ struct ufshpb_subregion *cp;
+ int bit_count, dword, bit_offset;
+ int count;
+
+ count = blk_rq_sectors(lrbp->cmd->request) >> sects_per_blk_shift;
+ ufshpb_get_bit_offset(hpb, subregion_offset,
+ &dword, &bit_offset);
+
+ do {
+ bit_count = min(count, BITS_PER_DWORD - bit_offset);
+
+ cb = hpb->region_tbl + region;
+ cp = cb->subregion_tbl + subregion;
+
+ ufshpb_l2p_entry_dirty_bit_set(hpb, cb, cp,
+ dword, bit_offset, bit_count);
+
+ bit_offset = 0;
+ dword++;
+
+ if (dword == hpb->dwords_per_subregion) {
+ dword = 0;
+ subregion++;
+
+ if (subregion == hpb->subregions_per_region) {
+ subregion = 0;
+ region++;
+ }
+ }
+
+ count -= bit_count;
+ } while (count);
+
+ BUG_ON(count < 0);
+}
+
+static struct ufshpb_rsp_info *ufshpb_rsp_info_get(struct ufshpb_lu *hpb)
+{
+ unsigned long flags;
+ struct ufshpb_rsp_info *rsp_info;
+
+ spin_lock_irqsave(&hpb->rsp_list_lock, flags);
+ rsp_info = list_first_entry_or_null(&hpb->lh_rsp_info_free,
+ struct ufshpb_rsp_info,
+ list_rsp_info);
+ if (!rsp_info) {
+ hpb_dbg(FAILURE_INFO, hpb, "No free rsp_info\n");
+ goto out;
+ }
+ list_del(&rsp_info->list_rsp_info);
+ memset(rsp_info, 0x00, sizeof(struct ufshpb_rsp_info));
+
+ INIT_LIST_HEAD(&rsp_info->list_rsp_info);
+out:
+
+ spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
+ return rsp_info;
+}
+
+static void ufshpb_rsp_info_put(struct ufshpb_lu *hpb,
+ struct ufshpb_rsp_info *rsp_info)
+{
+ unsigned long flags;
+
+ WARN_ON(!rsp_info);
+
+ spin_lock_irqsave(&hpb->rsp_list_lock, flags);
+ list_add_tail(&rsp_info->list_rsp_info, &hpb->lh_rsp_info_free);
+ spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
+}
+
+static void ufshpb_rsp_info_fill(struct ufshpb_lu *hpb,
+ struct ufshpb_rsp_info *rsp_info,
+ struct hpb_sense_data *sense_data)
+{
+ int num, idx;
+
+ rsp_info->type = HPB_REQ_REGION_UPDATE;
+ rsp_info->rsp_start = ktime_to_ns(ktime_get());
+
+ for (num = 0; num < sense_data->active_region_cnt; num++) {
+ idx = num * PER_ACTIVE_INFO_BYTES;
+ rsp_info->active_list.region[num] =
+ SHIFT_BYTE_1(sense_data->active_field[idx + 0]) |
+ SHIFT_BYTE_0(sense_data->active_field[idx + 1]);
+ rsp_info->active_list.subregion[num] =
+ SHIFT_BYTE_1(sense_data->active_field[idx + 2]) |
+ SHIFT_BYTE_0(sense_data->active_field[idx + 3]);
+ hpb_dbg(REGION_INFO, hpb,
+ "RSP UPIU: ACT[%d]: RG %d, SRG %d",
+ num + 1, rsp_info->active_list.region[num],
+ rsp_info->active_list.subregion[num]);
+ }
+ rsp_info->active_cnt = num;
+
+ for (num = 0; num < sense_data->inactive_region_cnt; num++) {
+ idx = num * PER_INACTIVE_INFO_BYTES;
+ rsp_info->inactive_list.region[num] =
+ SHIFT_BYTE_1(sense_data->inactive_field[idx + 0]) |
+ SHIFT_BYTE_0(sense_data->inactive_field[idx + 1]);
+ hpb_dbg(REGION_INFO, hpb,
+ "RSP UPIU: INACT[%d]: Region%d", num + 1,
+ rsp_info->inactive_list.region[num]);
+ }
+ rsp_info->inactive_cnt = num;
+}
+
+static struct ufshpb_req_info *ufshpb_req_info_get(struct ufshpb_lu *hpb)
+{
+ struct ufshpb_req_info *req_info;
+
+ spin_lock(&hpb->req_list_lock);
+ req_info = list_first_entry_or_null(&hpb->lh_req_info_free,
+ struct ufshpb_req_info,
+ list_req_info);
+ if (!req_info) {
+ hpb_dbg(FAILURE_INFO, hpb, "No free req_info\n");
+ goto out;
+ }
+
+ list_del(&req_info->list_req_info);
+ memset(req_info, 0x00, sizeof(struct ufshpb_req_info));
+
+ INIT_LIST_HEAD(&req_info->list_req_info);
+out:
+ spin_unlock(&hpb->req_list_lock);
+ return req_info;
+}
+
+static void ufshpb_req_info_put(struct ufshpb_lu *hpb,
+ struct ufshpb_req_info *req_info)
+{
+ WARN_ON(!req_info);
+
+ spin_lock(&hpb->req_list_lock);
+ list_add_tail(&req_info->list_req_info, &hpb->lh_req_info_free);
+ spin_unlock(&hpb->req_list_lock);
+}
+
+static inline bool ufshpb_is_read_lrbp(struct ufshcd_lrb *lrbp)
+{
+ if (lrbp->cmd->cmnd[0] == READ_10 || lrbp->cmd->cmnd[0] == READ_16)
+ return true;
+
+ return false;
+}
+
+static inline bool ufshpb_is_write_discard_lrbp(struct ufshcd_lrb *lrbp)
+{
+ if (lrbp->cmd->cmnd[0] == WRITE_10 || lrbp->cmd->cmnd[0] == WRITE_16
+ || lrbp->cmd->cmnd[0] == UNMAP)
+ return true;
+
+ return false;
+}
+
+static inline void ufshpb_get_pos_from_lpn(struct ufshpb_lu *hpb,
+ unsigned int lpn, int *region,
+ int *subregion, int *offset)
+{
+ int region_offset;
+
+ *region = lpn >> hpb->entries_per_region_shift;
+ region_offset = lpn & hpb->entries_per_region_mask;
+ *subregion = region_offset >> hpb->entries_per_subregion_shift;
+ *offset = region_offset & hpb->entries_per_subregion_mask;
+}
+
+static int ufshpb_subregion_dirty_bitmap_clean(struct ufshpb_lu *hpb,
+ struct ufshpb_subregion *cp)
+{
+ struct ufshpb_region *cb;
+
+ cb = hpb->region_tbl + cp->region;
+
+ /* if mctx is null, active block had been evicted out */
+ if (cb->region_state == REGION_INACTIVE || !cp->mctx) {
+ hpb_dbg(FAILURE_INFO, hpb, "RG_SRG(%d - %d) has been evicted",
+ cp->region, cp->subregion);
+ return -EINVAL;
+ }
+
+ memset(cp->mctx->ppn_dirty, 0x00,
+ hpb->entries_per_subregion >> bits_per_byte_shift);
+
+ return 0;
+}
+
+/*
+ * Change subregion_state to SUBREGION_CLEAN
+ */
+static void ufshpb_subregion_clean(struct ufshpb_lu *hpb,
+ struct ufshpb_subregion *cp)
+{
+ struct ufshpb_region *cb;
+
+ cb = hpb->region_tbl + cp->region;
+
+ /*
+ * If mctx is null or the state of region is not active,
+ * that means relavent region had been evicted out.
+ */
+ if (cb->region_state == REGION_INACTIVE || !cp->mctx) {
+ hpb_dbg(FAILURE_INFO, hpb, "RG_SRG %d - %d has been evicted\n",
+ cp->region, cp->subregion);
+ return;
+ }
+
+ cp->subregion_state = SUBREGION_CLEAN;
+}
+
+/*
+ * Change subregion_state to SUBREGION_DIRTY
+ */
+static void ufshpb_subregion_dirty(struct ufshpb_lu *hpb,
+ struct ufshpb_subregion *cp)
+{
+ struct ufshpb_region *cb;
+
+ cb = hpb->region_tbl + cp->region;
+
+ /* if mctx is null, active region had been evicted out */
+ if (cb->region_state == REGION_INACTIVE || !cp->mctx ||
+ cp->subregion_state == SUBREGION_DIRTY) {
+ return;
+ }
+ cp->subregion_state = SUBREGION_DIRTY;
+ cb->subregion_dirty_count++;
+}
+
+static int ufshpb_subregion_is_clean(struct ufshpb_lu *hpb,
+ struct ufshpb_subregion *cp)
+{
+ struct ufshpb_region *cb;
+ int is_clean;
+
+ cb = hpb->region_tbl + cp->region;
+ is_clean = true;
+
+ if (cb->region_state == REGION_INACTIVE) {
+ is_clean = false;
+ atomic64_inc(&hpb->status.region_miss);
+ } else if (cp->subregion_state != SUBREGION_CLEAN) {
+ is_clean = false;
+ atomic64_inc(&hpb->status.subregion_miss);
+
+ }
+
+ return is_clean;
+}
+
+static int ufshpb_make_region_inactive(struct ufshpb_lu *hpb,
+ int *victim_region)
+{
+
+ int region, ret;
+ struct ufshpb_region *victim_cb;
+ struct ufshpb_req_info *req_info;
+
+ req_info = ufshpb_req_info_get(hpb);
+ if (!req_info) {
+ hpb_dbg(FAILURE_INFO, hpb, "No req_info in mempool\n");
+ return -ENOSPC;
+ }
+
+ ret = ufshpb_evict_one_region(hpb, ®ion);
+ if (ret)
+ goto out;
+
+ victim_cb = hpb->region_tbl + region;
+ if (victim_cb->region_state != REGION_INACTIVE) {
+ ret = -EINVAL;
+ goto out;
+ }
+ req_info->type = HPB_REGION_DEACTIVATE;
+ req_info->region = region;
+ req_info->req_start_t = ktime_to_ns(ktime_get());
+
+ spin_lock(&hpb->req_list_lock);
+ list_add_tail(&req_info->list_req_info, &hpb->lh_req_info);
+ spin_unlock(&hpb->req_list_lock);
+
+ *victim_region = region;
+ return 0;
+out:
+ ufshpb_req_info_put(hpb, req_info);
+ return ret;
+}
+
+static int ufshpb_subregion_register(struct ufshpb_lu *hpb,
+ struct ufshpb_subregion *cp)
+{
+ struct ufshpb_req_info *req_info;
+ struct active_region_info *lru_info;
+ struct ufshpb_region *cb;
+ int deactivate_region_add;
+ int region;
+ int ret;
+
+ cb = hpb->region_tbl + cp->region;
+ lru_info = &hpb->lru_info;
+
+ if (cb->region_state == REGION_INACTIVE) {
+ if (atomic64_read(&lru_info->active_cnt) >=
+ lru_info->max_active_cnt) {
+ /*
+ * The number of active HPB Region is equal to the
+ * maximum active regions supported by device, the
+ * host is expected to issue a HPB WRITE BUFFER command
+ * to inactivate a region before activating a new region
+ */
+ ret = ufshpb_make_region_inactive(hpb, ®ion);
+ if (ret)
+ goto out;
+ hpb_dbg(REGION_INFO, hpb,
+ "Evict region %d, and activate region %d\n",
+ region, cp->region);
+ deactivate_region_add = 1;
+ }
+
+ if (ufshpb_region_mctx_alloc(hpb, cb))
+ goto out;
+ }
+
+ if (cp->subregion_state == SUBREGION_UNUSED ||
+ cp->subregion_state == SUBREGION_DIRTY) {
+ req_info = ufshpb_req_info_get(hpb);
+ if (!req_info) {
+ hpb_dbg(FAILURE_INFO, hpb, "No req_info in mempool\n");
+ if (!deactivate_region_add)
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ req_info->type = HPB_SUBREGION_ACTIVATE;
+ req_info->region = cp->region;
+ req_info->subregion = cp->subregion;
+ req_info->req_start_t = ktime_to_ns(ktime_get());
+
+ spin_lock(&hpb->req_list_lock);
+ list_add_tail(&req_info->list_req_info, &hpb->lh_req_info);
+ spin_unlock(&hpb->req_list_lock);
+
+ cp->subregion_state = SUBREGION_REGISTERED;
+ hpb_dbg(REGION_INFO, hpb,
+ "Add RG_SRG(%d - %d) to activate\n",
+ cp->region, cp->subregion);
+ }
+out:
+ return ret;
+}
+
+static void ufshpb_region_deactivate(struct ufshpb_lu *hpb,
+ struct ufshpb_req_info *req_info)
+{
+ struct ufshpb_region *cb;
+ unsigned char cmd[10];
+ int region;
+ int ret;
+
+ region = req_info->region;
+ cb = hpb->region_tbl + region;
+
+ if (region >= hpb->regions_per_lu) {
+ hpb_err("Region exceeds regions_per_lu\n");
+ return;
+ }
+
+ ufshpb_prepare_write_buf_cmd(cmd, region);
+
+ if (hpb->force_map_req_disable) {
+ hpb_notice("map disable - return");
+ return;
+ }
+
+ ret = ufshpb_execute_cmd(hpb, cmd);
+ if (ret) {
+ hpb_err("ufshpb_execute_cmd failed with error %d\n",
+ ret);
+ hpb->hba->ufshpb_state = HPB_FAILED;
+ schedule_work(&hpb->hba->ufshpb_eh_work);
+ }
+}
+
+static void ufshpb_subregion_activate(struct ufshpb_lu *hpb,
+ struct ufshpb_req_info *req_info)
+{
+ struct ufshpb_region *cb;
+ struct ufshpb_subregion *cp;
+ int region, subregion;
+ int ret;
+
+ region = req_info->region;
+ subregion = req_info->subregion;
+
+ if (region >= hpb->regions_per_lu) {
+ hpb_err("Region%d exceeds regions_per_lu\n",
+ region);
+ goto out;
+ }
+
+ cb = hpb->region_tbl + region;
+
+ if (subregion >= cb->subregion_count) {
+ hpb_err("Subregion%d exceeds its range %d - %d",
+ subregion, region, cb->subregion_count);
+ goto out;
+ }
+
+ cp = cb->subregion_tbl + subregion;
+
+ spin_lock(&hpb->hpb_lock);
+
+ if (list_empty(&cb->list_region)) {
+ spin_unlock(&hpb->hpb_lock);
+ hpb_warn("mctx wasn't assinged in Region%d\n", region);
+ goto out;
+ }
+
+ if (cp->subregion_state == SUBREGION_ISSUED ||
+ cp->subregion_state == SUBREGION_CLEAN) {
+ hpb_dbg(SCHEDULE_INFO, hpb,
+ "HPB BUFFER READ of RG_SRG (%d_%d) has been issued",
+ region, subregion);
+ spin_unlock(&hpb->hpb_lock);
+ goto out;
+ }
+
+ cp->subregion_state = SUBREGION_ISSUED;
+ ret = ufshpb_subregion_dirty_bitmap_clean(hpb, cp);
+ spin_unlock(&hpb->hpb_lock);
+ if (ret)
+ goto out;
+
+ if (hpb->force_map_req_disable) {
+ hpb_notice("map disable - return");
+ goto out;
+ }
+ ret = ufshpb_l2p_mapping_load(hpb, region, subregion,
+ cp->mctx, req_info->req_start_t,
+ req_info->workq_enter_t);
+ if (ret)
+ hpb_err("ufshpb_l2p_mapping_load failed with error %d",
+ ret);
+out:
+ return;
+}
+
+static inline struct ufshpb_map_ctx *ufshpb_mctx_get(struct ufshpb_lu *hpb,
+ int *err)
+{
+ struct ufshpb_map_ctx *mctx;
+
+ mctx = list_first_entry_or_null(&hpb->lh_map_ctx,
+ struct ufshpb_map_ctx, list_table);
+ if (mctx) {
+ list_del_init(&mctx->list_table);
+ hpb->free_mctx--;
+ *err = 0;
+ return mctx;
+ }
+ *err = -ENOMEM;
+
+ return NULL;
+}
+
+static inline void ufshpb_mctx_put(struct ufshpb_lu *hpb,
+ struct ufshpb_map_ctx *mctx)
+{
+ list_add(&mctx->list_table, &hpb->lh_map_ctx);
+ hpb->free_mctx++;
+}
+
+static inline void ufshpb_subregion_mctx_purge(struct ufshpb_lu *hpb,
+ struct ufshpb_subregion *cp,
+ int state)
+{
+ if (state == SUBREGION_UNUSED) {
+ ufshpb_mctx_put(hpb, cp->mctx);
+ cp->mctx = NULL;
+ }
+
+ cp->subregion_state = state;
+ atomic_set(&cp->read_counter, 0);
+}
+
+/*
+ * add new active region to active region info lru_info
+ */
+static inline void ufshpb_add_lru_info(struct active_region_info *lru_info,
+ struct ufshpb_region *cb)
+{
+ cb->region_state = REGION_ACTIVE;
+ cb->hit_count = 1;
+ list_add_tail(&cb->list_region, &lru_info->lru);
+ atomic64_inc(&lru_info->active_cnt);
+}
+
+static inline void ufshpb_hit_lru_info(struct active_region_info *lru_info,
+ struct ufshpb_region *cb)
+{
+ list_move_tail(&cb->list_region, &lru_info->lru);
+ if (cb->hit_count != 0xffffffff)
+ cb->hit_count++;
+}
+
+/*
+ * Remove one active region from active region info lru_info
+ */
+static inline void ufshpb_cleanup_lru_info(struct ufshpb_lu *hpb,
+ struct ufshpb_region *cb)
+{
+ struct active_region_info *lru_info;
+
+ lru_info = &hpb->lru_info;
+
+ list_del_init(&cb->list_region);
+ cb->region_state = REGION_INACTIVE;
+ cb->hit_count = 0;
+ atomic64_dec(&lru_info->active_cnt);
+}
+
+/*
+ * This function is only to change subregion_state to SUBREGION_CLEAN
+ */
+static void ufshpb_l2p_mapping_load_compl(struct ufshpb_lu *hpb,
+ struct ufshpb_map_req *map_req)
+{
+ map_req->req_end_t = ktime_to_ns(ktime_get());
+
+ hpb_trace(hpb, "%s: C RB %d - %d", DRIVER_NAME, map_req->region,
+ map_req->subregion);
+ hpb_dbg(SCHEDULE_INFO, hpb,
+ "LU%d HPB READ BUFFER COMPL RG_SRG (%d: %d), took %lldns\n",
+ hpb->lun, map_req->region, map_req->subregion,
+ map_req->req_end_t - map_req->req_start_t);
+ hpb_dbg(SCHEDULE_INFO, hpb,
+ "stage1 %lldns, stage2 %lldns, stage3 %lldns, stage4 %lldns",
+ map_req->workq_enter_t - map_req->req_start_t,
+ map_req->req_issue_t - map_req->workq_enter_t,
+ map_req->req_endio_t - map_req->req_issue_t,
+ map_req->req_end_t - map_req->req_endio_t);
+
+ spin_lock(&hpb->hpb_lock);
+ ufshpb_subregion_clean(hpb,
+ hpb->region_tbl[map_req->region].subregion_tbl +
+ map_req->subregion);
+ spin_unlock(&hpb->hpb_lock);
+}
+
+static void ufshpb_l2p_mapping_req_done(struct request *req,
+ blk_status_t status)
+{
+ struct ufs_hba *hba;
+ struct ufshpb_lu *hpb;
+ struct ufshpb_region *cb;
+ struct scsi_sense_hdr sshdr;
+ struct ufshpb_rsp_info *rsp_info;
+ struct ufshpb_map_req *map_req;
+ struct scsi_cmnd *scmd;
+ unsigned long flags;
+
+ map_req = (struct ufshpb_map_req *) req->end_io_data;
+
+ hpb = map_req->hpb;
+ hba = hpb->hba;
+ cb = hpb->region_tbl + map_req->region;
+ map_req->req_endio_t = ktime_to_ns(ktime_get());
+
+ if (hba->ufshpb_state != HPB_PRESENT)
+ goto free_map_req;
+
+ if (status) {
+ scmd = blk_mq_rq_to_pdu(req);
+ hpb_err("LU%d HPB READ BUFFER RG-SRG (%d: %d) failed\n",
+ map_req->lun, map_req->region, map_req->subregion);
+ hpb_err("SCSI Request result 0x%x, blk error status %d\n",
+ scsi_req(req)->result, status);
+
+ scsi_normalize_sense(scmd->sense_buffer, SCSI_SENSE_BUFFERSIZE,
+ &sshdr);
+ hpb_err("RSP_code 0x%x sense_key 0x%x ASC 0x%x ASCQ 0x%x\n",
+ sshdr.response_code, sshdr.sense_key, sshdr.asc,
+ sshdr.ascq);
+ hpb_err("byte4 %x byte5 %x byte6 %x additional_len %x\n",
+ sshdr.byte4, sshdr.byte5,
+ sshdr.byte6, sshdr.additional_length);
+ atomic64_inc(&hpb->status.rb_fail);
+ if (sshdr.sense_key == ILLEGAL_REQUEST) {
+ spin_lock(&hpb->hpb_lock);
+ if (cb->region_state == REGION_PINNED) {
+ if (map_req->retry_cnt < 5) {
+ /*FIXME, need to check ACS and ASCQ*/
+ /*
+ * If the HPB buffer Read request
+ * meets the UFS deivce GC, the
+ * device maybe returns ILLEGAL_REQUEST.
+ * For the case, we should retry later.
+ */
+ hpb_dbg(REGION_INFO, hpb,
+ "Retry pinned RG_SRG (%d - %d)",
+ map_req->region,
+ map_req->subregion);
+
+ INIT_LIST_HEAD(&map_req->list_map_req);
+ list_add_tail(&map_req->list_map_req,
+ &hpb->lh_map_req_retry);
+ spin_unlock(&hpb->hpb_lock);
+
+ schedule_delayed_work(&hpb->retry_work,
+ msecs_to_jiffies(5000));
+ return;
+
+ } else {
+ hpb_dbg(REGION_INFO, hpb,
+ "Mark pinned SRG %d-%d(dirty)",
+ map_req->region,
+ map_req->subregion);
+
+ ufshpb_subregion_dirty(hpb,
+ cb->subregion_tbl +
+ map_req->subregion);
+ spin_unlock(&hpb->hpb_lock);
+ }
+ } else {
+ /*
+ * If region which was read is not active
+ * in UFS device, the UFS device likely return
+ * ILLEGAL_REQUEST, for this case, we should
+ * deactivate this region in local cache.
+ */
+ spin_unlock(&hpb->hpb_lock);
+
+ rsp_info = ufshpb_rsp_info_get(hpb);
+ if (!rsp_info) {
+ hpb_warn("%s: %d No free rsp_info\n",
+ __func__, __LINE__);
+ goto free_map_req;
+ }
+
+ rsp_info->type = HPB_REQ_REGION_UPDATE;
+ rsp_info->rsp_start = ktime_to_ns(ktime_get());
+ rsp_info->active_cnt = 0;
+ rsp_info->inactive_cnt = 1;
+ rsp_info->inactive_list.region[0] =
+ map_req->region;
+
+ hpb_dbg(REGION_INFO, hpb,
+ "Normal RG %d is to deactivate\n",
+ map_req->region);
+
+ spin_lock_irqsave(&hpb->rsp_list_lock, flags);
+ list_add_tail(&rsp_info->list_rsp_info,
+ &hpb->lh_rsp_info);
+ spin_unlock_irqrestore(&hpb->rsp_list_lock,
+ flags);
+
+ schedule_work(&hpb->ufshpb_rsp_work);
+ }
+ } else {
+ /* FIXME, need to fix EIO error and timeout issue*/
+ ;
+ }
+ } else {
+ ufshpb_l2p_mapping_load_compl(hpb, map_req);
+ }
+free_map_req:
+ INIT_LIST_HEAD(&map_req->list_map_req);
+ spin_lock(&hpb->hpb_lock);
+ map_req->req = NULL;
+ scsi_req_free_cmd(scsi_req(req));
+ blk_put_request(req);
+ list_add_tail(&map_req->list_map_req, &hpb->lh_map_req_free);
+ spin_unlock(&hpb->hpb_lock);
+}
+
+static int ufshpb_execute_cmd(struct ufshpb_lu *hpb, unsigned char *cmd)
+{
+ struct scsi_sense_hdr sshdr;
+ struct scsi_device *sdp;
+ struct ufs_hba *hba;
+ int retries;
+ int ret = 0;
+
+ hba = hpb->hba;
+
+ sdp = hba->sdev_ufs_lu[hpb->lun];
+ if (!sdp) {
+ hpb_warn("%s UFSHPB cannot find scsi_device\n",
+ __func__);
+ return -ENODEV;
+ }
+
+ ret = scsi_device_get(sdp);
+ if (!ret && !scsi_device_online(sdp)) {
+ ret = -ENODEV;
+ scsi_device_put(sdp);
+ return ret;
+ }
+
+ for (retries = 3; retries > 0; --retries) {
+ ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
+ msecs_to_jiffies(30000), 3, 0, RQF_PM, NULL);
+ if (ret == 0)
+ break;
+ }
+
+ if (ret) {
+ if (driver_byte(ret) == DRIVER_SENSE &&
+ scsi_sense_valid(&sshdr)) {
+ switch (sshdr.sense_key) {
+ case ILLEGAL_REQUEST:
+ hpb_err("ILLEGAL REQUEST asc=0x%x ascq=0x%x\n",
+ sshdr.asc, sshdr.ascq);
+ break;
+ default:
+ hpb_err("Unknown return code = %x\n", ret);
+ break;
+ }
+ }
+ }
+ scsi_device_put(sdp);
+
+ return ret;
+}
+
+static int ufshpb_add_bio_page(struct ufshpb_lu *hpb, struct request_queue *q,
+ struct bio *bio, struct bio_vec *bvec,
+ struct ufshpb_map_ctx *mctx)
+{
+ struct page *page = NULL;
+ int i, ret = 0;
+
+ WARN_ON(!mctx);
+ WARN_ON(!bvec);
+ /*
+ * FIXME, below bvec array depth is samller than mpages_per_subregion.
+ */
+ bio_init(bio, bvec, hpb->mpages_per_subregion);
+
+ for (i = 0; i < hpb->mpages_per_subregion; i++) {
+ /* virt_to_page(p + (PAGE_SIZE * i)); */
+ page = mctx->m_page[i];
+ if (!page)
+ return -ENOMEM;
+
+ ret = bio_add_pc_page(q, bio, page, hpb->mpage_bytes, 0);
+
+ if (ret != hpb->mpage_bytes) {
+ hpb_err("Error ret %d", ret);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+static int ufshpb_l2p_mapping_load_req(struct ufshpb_lu *hpb,
+ struct request_queue *q,
+ struct ufshpb_map_req *map_req)
+{
+ struct request *req;
+ struct scsi_request *scsi_rq;
+ struct bio *bio;
+ unsigned char cmd[16] = { };
+ int alloc_len;
+ int ret;
+
+ bio = &map_req->bio;
+
+ ret = ufshpb_add_bio_page(hpb, q, bio, map_req->bvec, map_req->mctx);
+ if (ret) {
+ hpb_err("ufshpb_add_bio_page failed with error %d", ret);
+ return ret;
+ }
+
+ alloc_len = hpb->subregion_entry_size;
+ /*
+ * HPB Sub-Regions are equally sized except for
+ * the last one which is smaller if the last hpb
+ * Region is not an interger multiple of bHPBSubRegionSize.
+ */
+ if (map_req->region == (hpb->regions_per_lu - 1) &&
+ map_req->subregion == (hpb->subregions_in_last_region - 1))
+ alloc_len = hpb->last_subregion_entry_size;
+
+ ufshpb_prepare_read_buf_cmd(cmd, map_req->region, map_req->subregion,
+ alloc_len);
+ if (map_req->req == NULL) {
+ map_req->req = blk_get_request(q, REQ_OP_SCSI_IN, 0);
+ if (IS_ERR(map_req->req))
+ return PTR_ERR(map_req->req);
+ }
+ req = map_req->req;
+ scsi_rq = scsi_req(req);
+
+ blk_rq_append_bio(req, &bio);
+
+ scsi_req_init(scsi_rq);
+
+ scsi_rq->cmd_len = COMMAND_SIZE(cmd[0]);
+ memcpy(scsi_rq->cmd, cmd, scsi_rq->cmd_len);
+ req->timeout = msecs_to_jiffies(30000);
+ req->end_io_data = (void *)map_req;
+
+ hpb_dbg(SCHEDULE_INFO, hpb,
+ "ISSUE READ_BUFFER : %d - %d retry = %d",
+ map_req->region, map_req->subregion, map_req->retry_cnt);
+ hpb_trace(hpb, "%s: I RB %d - %d", DRIVER_NAME, map_req->region,
+ map_req->subregion);
+
+ blk_execute_rq_nowait(q, NULL, req, 1, ufshpb_l2p_mapping_req_done);
+ map_req->req_issue_t = ktime_to_ns(ktime_get());
+
+ atomic64_inc(&hpb->status.map_req_cnt);
+
+ return 0;
+}
+
+static int ufshpb_l2p_mapping_load(struct ufshpb_lu *hpb, int region,
+ int subregion, struct ufshpb_map_ctx *mctx,
+ __u64 start_t, __u64 workq_enter_t)
+{
+ struct ufshpb_map_req *map_req;
+ struct request_queue *q;
+
+ spin_lock(&hpb->hpb_lock);
+ map_req = list_first_entry_or_null(&hpb->lh_map_req_free,
+ struct ufshpb_map_req,
+ list_map_req);
+ if (!map_req) {
+ hpb_dbg(FAILURE_INFO, hpb, "There is no free map_req");
+ spin_unlock(&hpb->hpb_lock);
+ return -ENOMEM;
+ }
+ list_del(&map_req->list_map_req);
+ memset(map_req, 0x00, sizeof(struct ufshpb_map_req));
+
+ spin_unlock(&hpb->hpb_lock);
+
+ map_req->hpb = hpb;
+ map_req->retry_cnt = 0;
+ map_req->region = region;
+ map_req->subregion = subregion;
+ map_req->mctx = mctx;
+ map_req->lun = hpb->lun;
+ map_req->req_start_t = start_t;
+ map_req->workq_enter_t = workq_enter_t;
+
+ q = hpb->hba->sdev_ufs_lu[hpb->lun]->request_queue;
+ return ufshpb_l2p_mapping_load_req(hpb, q, map_req);
+}
+
+static inline int ufshpb_region_mctx_evict(struct ufshpb_lu *hpb,
+ struct ufshpb_region *cb)
+{
+ struct ufshpb_subregion *cp;
+ int subregion;
+
+ if (cb->region_state == REGION_PINNED) {
+ /*
+ * Pinned active-block should not drop-out.
+ * But if so, it would treat error as critical,
+ * and it will run ufshpb_eh_work
+ */
+ hpb_warn("Trying to evict HPB pinned region\n");
+ return -ENOMEDIUM;
+ }
+
+ if (list_empty(&cb->list_region)) {
+ /* The region being evicted has been inactive */
+ return 0;
+ }
+
+
+ hpb_dbg(REGION_INFO, hpb,
+ "\x1b[41m\x1b[33m EVICT region: %d \x1b[0m", cb->region);
+ hpb_trace(hpb, "%s: EVIC RG: %d", DRIVER_NAME, cb->region);
+
+ ufshpb_cleanup_lru_info(hpb, cb);
+
+ atomic64_inc(&hpb->status.region_evict);
+ for (subregion = 0; subregion < cb->subregion_count; subregion++) {
+ cp = cb->subregion_tbl + subregion;
+ ufshpb_subregion_mctx_purge(hpb, cp, SUBREGION_UNUSED);
+ }
+ return 0;
+}
+
+static struct
+ufshpb_region *ufshpb_victim_region_select(struct active_region_info *lru_info)
+{
+ struct ufshpb_region *cb;
+ struct ufshpb_region *victim_cb = NULL;
+ u32 hit_count = 0xffffffff;
+
+ switch (lru_info->selection_type) {
+ case LRU:
+ /*
+ * Choose first active in list to evict
+ */
+ victim_cb = list_first_entry(&lru_info->lru,
+ struct ufshpb_region, list_region);
+ break;
+ case LFU:
+ /*
+ * Choose active region with the lowest hit_count to evict
+ */
+ list_for_each_entry(cb, &lru_info->lru, list_region) {
+ if (hit_count > cb->hit_count) {
+ hit_count = cb->hit_count;
+ victim_cb = cb;
+ }
+ }
+ break;
+ default:
+ break;
+ }
+
+ return victim_cb;
+}
+
+static int ufshpb_evict_one_region(struct ufshpb_lu *hpb, int *region)
+{
+ struct ufshpb_region *victim_cb;
+ struct active_region_info *lru_info;
+
+ lru_info = &hpb->lru_info;
+
+ victim_cb = ufshpb_victim_region_select(lru_info);
+ if (!victim_cb) {
+ hpb_warn("UFSHPB victim_cb is NULL\n");
+ return -EIO;
+ }
+ hpb_trace(hpb, "%s: VT RG %d", DRIVER_NAME, victim_cb->region);
+ hpb_dbg(REGION_INFO, hpb, "LU%d HPB will release active region %d\n",
+ hpb->lun, victim_cb->region);
+ *region = victim_cb->region;
+
+ return ufshpb_region_mctx_evict(hpb, victim_cb);
+}
+
+static inline int __ufshpb_region_mctx_alloc(struct ufshpb_lu *hpb,
+ struct ufshpb_region *cb)
+{
+ int subregion;
+ int err = 0;
+
+
+ hpb_dbg(REGION_INFO, hpb,
+ "\x1b[44m\x1b[32m Activate region: %d \x1b[0m",
+ cb->region);
+ hpb_trace(hpb, "%s: ACT RG: %d", DRIVER_NAME, cb->region);
+
+ for (subregion = 0; subregion < cb->subregion_count; subregion++) {
+ struct ufshpb_subregion *cp;
+
+ cp = cb->subregion_tbl + subregion;
+
+ cp->mctx = ufshpb_mctx_get(hpb, &err);
+ if (err) {
+ hpb_dbg(FAILURE_INFO, hpb,
+ "Failed to get mctx for SRG %d, free mctxs %d",
+ subregion, hpb->free_mctx);
+ goto out;
+ }
+ }
+
+ atomic64_inc(&hpb->status.region_add);
+out:
+ return err;
+}
+
+static int ufshpb_region_mctx_alloc(struct ufshpb_lu *hpb,
+ struct ufshpb_region *cb)
+{
+ struct active_region_info *lru_info;
+ int ret;
+
+ lru_info = &hpb->lru_info;
+ /*
+ * if already region is added to lru_list,
+ * just initiate the information of lru.
+ * because the region already has the mapping ctx.
+ * (!list_empty(&cb->list_region) equals to region->state=active...)
+ */
+ if (!list_empty(&cb->list_region)) {
+ hpb_dbg(REGION_INFO, hpb,
+ "LU%d Region %d already in lur_list",
+ hpb->lun, cb->region);
+ ufshpb_hit_lru_info(lru_info, cb);
+ return 0;
+ }
+
+ if (cb->region_state == REGION_INACTIVE)
+ hpb_dbg(REGION_INFO, hpb, "Region%d is inactive", cb->region);
+
+ ret = __ufshpb_region_mctx_alloc(hpb, cb);
+ if (ret) {
+ hpb_warn("UFSHPB mctx allocation failed\n");
+ return -ENOMEM;
+ }
+
+ ufshpb_add_lru_info(lru_info, cb);
+
+ return 0;
+}
+
+static int ufshpb_region_mctx_evict_add(struct ufshpb_lu *hpb,
+ struct ufshpb_rsp_info *rsp_info)
+{
+ struct active_region_info *lru_info;
+ struct ufshpb_subregion *cp;
+ struct ufshpb_region *cb;
+ int region, subregion;
+ int active_cnt;
+ int iter;
+
+ lru_info = &hpb->lru_info;
+ hpb_dbg(REGION_INFO, hpb,
+ "LU%d now has %lld active regions", hpb->lun,
+ atomic64_read(&lru_info->active_cnt));
+
+ for (iter = 0; iter < rsp_info->inactive_cnt; iter++) {
+ region = rsp_info->inactive_list.region[iter];
+ cb = hpb->region_tbl + region;
+
+ spin_lock(&hpb->hpb_lock);
+ if (ufshpb_region_mctx_evict(hpb, cb))
+ goto out;
+
+ spin_unlock(&hpb->hpb_lock);
+ }
+
+ active_cnt = rsp_info->active_cnt;
+ for (iter = 0; iter < active_cnt; iter++) {
+ region = rsp_info->active_list.region[iter];
+ cb = hpb->region_tbl + region;
+
+ spin_lock(&hpb->hpb_lock);
+ switch (hpb->hba->dev_info.hpb_control_mode) {
+ case HPB_HOST_CTRL_MODE:
+ /*
+ * For the HPB host control mode, we just dirty
+ * this subregion, and remove it from recommended
+ * subregion array
+ */
+ subregion = rsp_info->active_list.subregion[iter];
+ cp = cb->subregion_tbl + subregion;
+ ufshpb_subregion_dirty(hpb, cp);
+ rsp_info->active_cnt--;
+ break;
+ case HPB_DEV_CTRL_MODE:
+ if (atomic64_read(&lru_info->active_cnt) >=
+ lru_info->max_active_cnt) {
+ int victim_region;
+
+ if (ufshpb_evict_one_region(hpb,
+ &victim_region))
+ goto out;
+ }
+ if (ufshpb_region_mctx_alloc(hpb, cb))
+ goto out;
+ break;
+ default:
+ hpb_err("Unknown HPB control mode\n");
+ break;
+ }
+ spin_unlock(&hpb->hpb_lock);
+ }
+
+ return 0;
+out:
+ spin_unlock(&hpb->hpb_lock);
+ return -ENOMEM;
+}
+
+static void ufshpb_recommended_l2p_update(struct ufshpb_lu *hpb,
+ struct ufshpb_rsp_info *rsp_info)
+{
+ struct ufshpb_region *cb;
+ struct ufshpb_subregion *cp;
+ int region, subregion;
+ int iter;
+ int ret;
+
+ /*
+ * Before HPB Read Buffer CMD for active region/subregion,
+ * prepare mctx for each subregion
+ */
+ ret = ufshpb_region_mctx_evict_add(hpb, rsp_info);
+ if (ret) {
+ hpb_err("region mctx evict/load failed\n");
+ goto wakeup_ee_worker;
+ }
+
+ for (iter = 0; iter < rsp_info->active_cnt; iter++) {
+ region = rsp_info->active_list.region[iter];
+ subregion = rsp_info->active_list.subregion[iter];
+ cb = hpb->region_tbl + region;
+
+ if (region >= hpb->regions_per_lu ||
+ subregion >= cb->subregion_count) {
+ hpb_err("ufshpb issue-map %d - %d range error",
+ region, subregion);
+ goto wakeup_ee_worker;
+ }
+
+ cp = cb->subregion_tbl + subregion;
+
+ /* if subregion_state set SUBREGION_ISSUED,
+ * active_page has already been added to list,
+ * so it just ends function.
+ */
+
+ spin_lock(&hpb->hpb_lock);
+ if (cp->subregion_state == SUBREGION_ISSUED) {
+ hpb_dbg(SCHEDULE_INFO, hpb,
+ "HPB buffer Read for region %d has been issued",
+ region);
+ spin_unlock(&hpb->hpb_lock);
+ continue;
+ }
+
+ cp->subregion_state = SUBREGION_ISSUED;
+
+ ret = ufshpb_subregion_dirty_bitmap_clean(hpb, cp);
+
+ spin_unlock(&hpb->hpb_lock);
+
+ if (ret)
+ continue;
+
+ if (hpb->force_map_req_disable) {
+ hpb_notice("map disable - return");
+ return;
+ }
+
+ ret = ufshpb_l2p_mapping_load(hpb, region, subregion,
+ cp->mctx, rsp_info->rsp_start,
+ rsp_info->rsp_tasklet_enter);
+ if (ret) {
+ hpb_err("ufshpb_l2p_mapping_load failed with error %d",
+ ret);
+ goto wakeup_ee_worker;
+ }
+ }
+ return;
+
+wakeup_ee_worker:
+ hpb->hba->ufshpb_state = HPB_FAILED;
+ schedule_work(&hpb->hba->ufshpb_eh_work);
+}
+
+static inline void ufshpb_add_subregion_to_req_list(struct ufshpb_lu *hpb,
+ struct ufshpb_subregion *cp)
+{
+ list_add_tail(&cp->list_subregion, &hpb->lh_subregion_req);
+ cp->subregion_state = SUBREGION_ISSUED;
+}
+
+static int ufshpb_execute_req(struct ufshpb_lu *hpb, unsigned char *cmd,
+ struct ufshpb_subregion *cp)
+{
+ struct request_queue *q;
+ struct request *req;
+ struct scsi_request *scsi_rq;
+ struct scsi_sense_hdr sshdr;
+ struct scsi_device *sdp;
+ struct bio bio;
+ struct bio *bio_p = &bio;
+ struct bio_vec *bvec;
+ struct ufs_hba *hba;
+ char sense[SCSI_SENSE_BUFFERSIZE];
+ unsigned long flags;
+ int ret = 0;
+
+ hba = hpb->hba;
+ bvec = kmalloc_array(hpb->mpages_per_subregion, sizeof(struct bio_vec),
+ GFP_KERNEL);
+ if (!bvec)
+ return -ENOMEM;
+
+ sdp = hba->sdev_ufs_lu[hpb->lun];
+ if (!sdp) {
+ hpb_warn("%s UFSHPB cannot find scsi_device\n",
+ __func__);
+ ret = -ENODEV;
+ goto mem_free_out;
+ }
+
+ spin_lock_irqsave(hba->host->host_lock, flags);
+ ret = scsi_device_get(sdp);
+ if (!ret && !scsi_device_online(sdp)) {
+ spin_unlock_irqrestore(hba->host->host_lock, flags);
+ ret = -ENODEV;
+ scsi_device_put(sdp);
+ goto mem_free_out;
+ }
+ spin_unlock_irqrestore(hba->host->host_lock, flags);
+
+ q = sdp->request_queue;
+
+ ret = ufshpb_add_bio_page(hpb, q, &bio, bvec, cp->mctx);
+ if (ret) {
+ scsi_device_put(sdp);
+ goto mem_free_out;
+ }
+
+ req = blk_get_request(q, REQ_OP_SCSI_IN, 0);
+ if (IS_ERR(req))
+ return PTR_ERR(req);
+
+ blk_rq_append_bio(req, &bio_p);
+
+ scsi_rq = scsi_req(req);
+ scsi_req_init(scsi_rq);
+
+ scsi_rq->cmd_len = COMMAND_SIZE(cmd[0]);
+ memcpy(scsi_rq->cmd, cmd, scsi_rq->cmd_len);
+
+ scsi_rq->sense = sense;
+ scsi_rq->sense_len = SCSI_SENSE_BUFFERSIZE;
+
+ scsi_rq->retries = 3;
+ req->timeout = msecs_to_jiffies(10000);
+
+ blk_execute_rq(q, NULL, req, 1);
+ if (scsi_rq->result) {
+ ret = scsi_rq->result;
+ scsi_normalize_sense(scsi_rq->sense, SCSI_SENSE_BUFFERSIZE,
+ &sshdr);
+ hpb_err("code 0x%x sense_key 0x%x asc 0x%x ascq 0x%x",
+ sshdr.response_code, sshdr.sense_key, sshdr.asc,
+ sshdr.ascq);
+ hpb_err("byte4 0x%x byte5 0x%x byte6 0x%x additional_len 0x%x",
+ sshdr.byte4, sshdr.byte5, sshdr.byte6,
+ sshdr.additional_length);
+ spin_lock_bh(&hpb->hpb_lock);
+ ufshpb_subregion_dirty(hpb, cp);
+ spin_unlock_bh(&hpb->hpb_lock);
+ } else {
+ ret = 0;
+ spin_lock_bh(&hpb->hpb_lock);
+ ufshpb_subregion_dirty_bitmap_clean(hpb, cp);
+ spin_unlock_bh(&hpb->hpb_lock);
+ }
+ scsi_device_put(sdp);
+ blk_put_request(req);
+mem_free_out:
+ kfree(bvec);
+ return ret;
+}
+
+static int ufshpb_issue_map_req_from_list(struct ufshpb_lu *hpb)
+{
+ struct ufshpb_subregion *cp, *next_cp;
+ int alloc_len;
+ int ret;
+
+ LIST_HEAD(req_list);
+
+ spin_lock_bh(&hpb->hpb_lock);
+ list_splice_init(&hpb->lh_subregion_req,
+ &req_list);
+ spin_unlock_bh(&hpb->hpb_lock);
+
+ list_for_each_entry_safe(cp, next_cp, &req_list, list_subregion) {
+ unsigned char cmd[10] = { 0 };
+
+ alloc_len = hpb->subregion_entry_size;
+ /*
+ * HPB Sub-Regions are equally sized except for
+ * the last one which is smaller if the last hpb
+ * Region is not an interger multiple of bHPBSubRegionSize.
+ */
+ if (cp->region == (hpb->regions_per_lu - 1) &&
+ cp->subregion == (hpb->subregions_in_last_region - 1))
+ alloc_len = hpb->last_subregion_entry_size;
+
+ ufshpb_prepare_read_buf_cmd(cmd, cp->region, cp->subregion,
+ alloc_len);
+
+ hpb_dbg(SCHEDULE_INFO, hpb,
+ "RG_SRG(%d -%d) ISSUE READ_BUFFER",
+ cp->region, cp->subregion);
+
+ hpb_trace(hpb, "%s: I RB %d - %d", DRIVER_NAME,
+ cp->region, cp->subregion);
+
+ ret = ufshpb_execute_req(hpb, cmd, cp);
+ if (ret < 0) {
+ hpb_err("RG_SRG(%d - %d) HPB READ BUFFER failed %d\n",
+ cp->region, cp->subregion, ret);
+ continue;
+ }
+
+
+ hpb_trace(hpb, "%s: C RB %d - %d", DRIVER_NAME,
+ cp->region, cp->subregion);
+
+ hpb_dbg(REGION_INFO, hpb,
+ "RG_SRG(%d -%d) HPB READ_BUFFER COML",
+ cp->region, cp->subregion);
+
+ spin_lock_bh(&hpb->hpb_lock);
+ ufshpb_subregion_clean(hpb, cp);
+ list_del_init(&cp->list_subregion);
+ spin_unlock_bh(&hpb->hpb_lock);
+ }
+
+ return 0;
+}
+
+static void ufshpb_sync_workq_fn(struct work_struct *work)
+{
+ struct ufshpb_lu *hpb;
+ int ret;
+
+ hpb = container_of(work, struct ufshpb_lu, ufshpb_sync_work);
+ hpb_dbg(SCHEDULE_INFO, hpb, "sync worker start\n");
+
+ if (!list_empty(&hpb->lh_subregion_req)) {
+ ret = ufshpb_issue_map_req_from_list(hpb);
+ /*
+ * If its function failed at init time,
+ * ufshpb-device will request map-req,
+ * so it is not critical-error, and just
+ * finish work-handler
+ */
+ if (ret)
+ hpb_err("Issue map_req failed with ret %d\n",
+ ret);
+ }
+
+ hpb_dbg(SCHEDULE_INFO, hpb, "sync worker end\n");
+}
+
+static void ufshpb_req_workq_fn(struct work_struct *work)
+{
+ struct ufshpb_lu *hpb;
+ struct ufshpb_req_info *req_info;
+
+ LIST_HEAD(req_list);
+
+ hpb = container_of(work, struct ufshpb_lu, ufshpb_req_work);
+ spin_lock(&hpb->req_list_lock);
+ if (list_empty(&hpb->lh_req_info)) {
+ spin_unlock(&hpb->req_list_lock);
+ return;
+ }
+ list_splice_init(&hpb->lh_req_info, &req_list);
+ spin_unlock(&hpb->req_list_lock);
+
+ while (1) {
+ req_info = list_first_entry_or_null(&req_list,
+ struct ufshpb_req_info,
+ list_req_info);
+ if (!req_info) {
+ hpb_dbg(FAILURE_INFO, hpb, "req_info list is NULL\n");
+ break;
+ }
+
+ list_del(&req_info->list_req_info);
+
+ req_info->workq_enter_t = ktime_to_ns(ktime_get());
+
+ switch (req_info->type) {
+ case HPB_SUBREGION_ACTIVATE:
+ ufshpb_subregion_activate(hpb, req_info);
+ hpb_dbg(SCHEDULE_INFO, hpb,
+ "Activate RG_SRG(%d - %d) done\n",
+ req_info->region, req_info->subregion);
+ break;
+ case HPB_REGION_DEACTIVATE:
+ ufshpb_region_deactivate(hpb, req_info);
+ hpb_dbg(SCHEDULE_INFO, hpb,
+ "Deactivate RG_SRG(%d - %d) done\n",
+ req_info->region, req_info->subregion);
+ break;
+ default:
+ break;
+ }
+
+ ufshpb_req_info_put(hpb, req_info);
+ }
+}
+
+static void ufshpb_retry_workq_fn(struct work_struct *work)
+{
+ struct ufshpb_lu *hpb;
+ struct delayed_work *dwork = to_delayed_work(work);
+ struct ufshpb_map_req *map_req;
+ struct request_queue *q;
+ int ret = 0;
+
+ LIST_HEAD(retry_list);
+
+ hpb = container_of(dwork, struct ufshpb_lu, retry_work);
+ hpb_dbg(SCHEDULE_INFO, hpb, "Retry workq start\n");
+
+ spin_lock_bh(&hpb->hpb_lock);
+ list_splice_init(&hpb->lh_map_req_retry, &retry_list);
+ spin_unlock_bh(&hpb->hpb_lock);
+
+ while (1) {
+ map_req = list_first_entry_or_null(&retry_list,
+ struct ufshpb_map_req,
+ list_map_req);
+ if (!map_req) {
+ hpb_dbg(SCHEDULE_INFO, hpb, "There is no map_req\n");
+ break;
+ }
+ list_del(&map_req->list_map_req);
+
+ map_req->retry_cnt++;
+
+ q = hpb->hba->sdev_ufs_lu[hpb->lun]->request_queue;
+ ret = ufshpb_l2p_mapping_load_req(hpb, q,
+ map_req);
+ if (ret) {
+ hpb_err("ufshpb_l2p_mapping_load_req error %d",
+ ret);
+ goto wakeup_ee_worker;
+ }
+ }
+ hpb_dbg(SCHEDULE_INFO, hpb, "Retry workq end\n");
+ return;
+
+wakeup_ee_worker:
+ hpb->hba->ufshpb_state = HPB_FAILED;
+ schedule_work(&hpb->hba->ufshpb_eh_work);
+}
+
+static void ufshpb_rsp_workq_fn(struct work_struct *work)
+{
+ struct ufshpb_lu *hpb;
+ struct ufshpb_rsp_info *rsp_info;
+ unsigned long flags;
+
+ hpb = container_of(work, struct ufshpb_lu, ufshpb_rsp_work);
+
+ while (1) {
+ spin_lock_irqsave(&hpb->rsp_list_lock, flags);
+ rsp_info = list_first_entry_or_null(&hpb->lh_rsp_info,
+ struct ufshpb_rsp_info,
+ list_rsp_info);
+ if (!rsp_info) {
+ spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
+ break;
+ }
+
+ list_del_init(&rsp_info->list_rsp_info);
+ spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
+
+ rsp_info->rsp_tasklet_enter = ktime_to_ns(ktime_get());
+ switch (rsp_info->type) {
+ case HPB_REQ_REGION_UPDATE:
+ ufshpb_recommended_l2p_update(hpb, rsp_info);
+ break;
+ default:
+ break;
+ }
+ ufshpb_rsp_info_put(hpb, rsp_info);
+ }
+}
+
+static void ufshpb_init_constant(void)
+{
+ sects_per_blk_shift = ffs(UFS_LOGICAL_BLOCK_SIZE) - ffs(SECTOR_SIZE);
+ bits_per_dword_shift = ffs(BITS_PER_DWORD) - 1;
+ bits_per_dword_mask = BITS_PER_DWORD - 1;
+ bits_per_byte_shift = ffs(BITS_PER_BYTE) - 1;
+}
+
+static int ufshpb_init_pinned_region(struct ufshpb_lu *hpb,
+ struct ufshpb_region *cb)
+{
+ struct ufshpb_subregion *cp;
+ int subregion, j;
+ int err = 0;
+
+ for (subregion = 0; subregion < cb->subregion_count; subregion++) {
+ cp = cb->subregion_tbl + subregion;
+
+ cp->mctx = ufshpb_mctx_get(hpb, &err);
+ if (err) {
+ hpb_err("Failed to get mctx for pinned SRG %d\n",
+ subregion);
+ goto release;
+ }
+ spin_lock_bh(&hpb->hpb_lock);
+ ufshpb_add_subregion_to_req_list(hpb, cp);
+ spin_unlock_bh(&hpb->hpb_lock);
+ }
+
+ cb->region_state = REGION_PINNED;
+ return 0;
+
+release:
+ for (j = 0; j < subregion; j++) {
+ cp = cb->subregion_tbl + j;
+ ufshpb_mctx_put(hpb, cp->mctx);
+ }
+
+ return err;
+}
+
+static inline bool ufshpb_is_pinned_region(struct ufshpb_lu_desc *lu_desc,
+ int region)
+{
+ if (lu_desc->lu_hpb_pinned_end_offset != -1 &&
+ region >= lu_desc->hpb_pinned_region_startidx &&
+ region <= lu_desc->lu_hpb_pinned_end_offset)
+ return true;
+
+ return false;
+}
+
+static inline void ufshpb_init_jobs(struct ufshpb_lu *hpb)
+{
+ INIT_WORK(&hpb->ufshpb_sync_work, ufshpb_sync_workq_fn);
+ INIT_WORK(&hpb->ufshpb_req_work, ufshpb_req_workq_fn);
+ INIT_DELAYED_WORK(&hpb->retry_work, ufshpb_retry_workq_fn);
+ INIT_WORK(&hpb->ufshpb_rsp_work, ufshpb_rsp_workq_fn);
+}
+
+static inline void ufshpb_cancel_jobs(struct ufshpb_lu *hpb)
+{
+ cancel_work_sync(&hpb->ufshpb_sync_work);
+ cancel_work_sync(&hpb->ufshpb_req_work);
+ cancel_delayed_work_sync(&hpb->retry_work);
+ cancel_work_sync(&hpb->ufshpb_rsp_work);
+}
+
+static void ufshpb_init_subregion_tbl(struct ufshpb_lu *hpb,
+ struct ufshpb_region *cb)
+{
+ int subregion;
+
+ for (subregion = 0; subregion < cb->subregion_count; subregion++) {
+ struct ufshpb_subregion *cp = cb->subregion_tbl + subregion;
+
+ cp->region = cb->region;
+ cp->subregion = subregion;
+ cp->subregion_state = SUBREGION_UNUSED;
+ }
+}
+
+static int ufshpb_alloc_subregion_tbl(struct ufshpb_lu *hpb,
+ struct ufshpb_region *cb,
+ int subregion_count)
+{
+ cb->subregion_tbl =
+ kcalloc(subregion_count, sizeof(struct ufshpb_subregion),
+ GFP_KERNEL);
+ if (!cb->subregion_tbl)
+ return -ENOMEM;
+
+ cb->subregion_count = subregion_count;
+
+ return 0;
+}
+
+static void ufshpb_mctx_mempool_remove(struct ufshpb_lu *hpb)
+{
+ struct ufshpb_map_ctx *mctx, *next;
+ int i;
+
+ list_for_each_entry_safe(mctx, next, &hpb->lh_map_ctx, list_table) {
+ for (i = 0; i < hpb->mpages_per_subregion; i++)
+ __free_page(mctx->m_page[i]);
+
+ vfree(mctx->ppn_dirty);
+ kfree(mctx->m_page);
+ kfree(mctx);
+ alloc_mctx--;
+ }
+}
+
+static int ufshpb_mctx_mempool_init(struct ufshpb_lu *hpb, int num_regions,
+ int subregions_per_region, int entry_count,
+ int entry_byte)
+{
+ int i, j, k;
+ struct ufshpb_map_ctx *mctx = NULL;
+
+ INIT_LIST_HEAD(&hpb->lh_map_ctx);
+
+ for (i = 0; i < num_regions * subregions_per_region; i++) {
+ mctx = kzalloc(sizeof(struct ufshpb_map_ctx), GFP_KERNEL);
+ if (!mctx)
+ goto release_mem;
+
+ mctx->m_page = kcalloc(hpb->mpages_per_subregion,
+ sizeof(struct page *),
+ GFP_KERNEL);
+ if (!mctx->m_page)
+ goto release_mem;
+
+ mctx->ppn_dirty = vzalloc(entry_count >> bits_per_byte_shift);
+ if (!mctx->ppn_dirty)
+ goto release_mem;
+
+ for (j = 0; j < hpb->mpages_per_subregion; j++) {
+ mctx->m_page[j] = alloc_page(GFP_KERNEL | __GFP_ZERO);
+ if (!mctx->m_page[j]) {
+ for (k = 0; k < j; k++)
+ kfree(mctx->m_page[k]);
+ goto release_mem;
+ }
+ }
+
+ INIT_LIST_HEAD(&mctx->list_table);
+ list_add(&mctx->list_table, &hpb->lh_map_ctx);
+ hpb->free_mctx++;
+ }
+
+ alloc_mctx = num_regions * subregions_per_region;
+ hpb_info("LU%d amount of mctx %d\n", hpb->lun, hpb->free_mctx);
+ return 0;
+
+release_mem:
+ /*
+ * mctxs already added in lh_map_ctx will be removed
+ * in the caller function.
+ */
+ if (mctx) {
+ kfree(mctx->m_page);
+ vfree(mctx->ppn_dirty);
+ kfree(mctx);
+ }
+ return -ENOMEM;
+}
+
+static inline void ufshpb_req_mempool_remove(struct ufshpb_lu *hpb)
+{
+ kfree(hpb->rsp_info);
+ kfree(hpb->req_info);
+ kfree(hpb->map_req);
+}
+
+static int ufshpb_req_mempool_init(struct ufshpb_lu *hpb, int num_pinned_rgn,
+ int queue_depth)
+{
+ struct ufs_hba *hba;
+ struct ufshpb_rsp_info *rsp_info = NULL;
+ struct ufshpb_req_info *req_info = NULL;
+ struct ufshpb_map_req *map_req = NULL;
+ int i, map_req_cnt = 0;
+
+ hba = hpb->hba;
+
+ if (!queue_depth) {
+ queue_depth = hba->nutrs;
+ hpb_info("LU%d private queue depth is 0\n", hpb->lun);
+ hpb_info("LU%d HPB Req uses Shared Queue depth %d\n",
+ hpb->lun, queue_depth);
+ }
+
+ INIT_LIST_HEAD(&hpb->lh_rsp_info);
+ INIT_LIST_HEAD(&hpb->lh_req_info);
+ INIT_LIST_HEAD(&hpb->lh_rsp_info_free);
+ INIT_LIST_HEAD(&hpb->lh_map_req_free);
+ INIT_LIST_HEAD(&hpb->lh_map_req_retry);
+ INIT_LIST_HEAD(&hpb->lh_req_info_free);
+
+ hpb->rsp_info = kcalloc(queue_depth, sizeof(struct ufshpb_rsp_info),
+ GFP_KERNEL);
+ if (!hpb->rsp_info)
+ goto out;
+
+ hpb->req_info = kcalloc(queue_depth, sizeof(struct ufshpb_req_info),
+ GFP_KERNEL);
+ if (!hpb->req_info)
+ goto out_free_rsp;
+
+ map_req_cnt = max(hpb->subregions_per_lu, queue_depth);
+
+ hpb->map_req = kcalloc(map_req_cnt, sizeof(struct ufshpb_map_req),
+ GFP_KERNEL);
+ if (!hpb->map_req)
+ goto out_free_req;
+
+ for (i = 0; i < queue_depth; i++) {
+ rsp_info = hpb->rsp_info + i;
+ req_info = hpb->req_info + i;
+ INIT_LIST_HEAD(&rsp_info->list_rsp_info);
+ INIT_LIST_HEAD(&req_info->list_req_info);
+ list_add_tail(&rsp_info->list_rsp_info, &hpb->lh_rsp_info_free);
+ list_add_tail(&req_info->list_req_info, &hpb->lh_req_info_free);
+ }
+
+ for (i = 0; i < map_req_cnt; i++) {
+ map_req = hpb->map_req + i;
+ INIT_LIST_HEAD(&map_req->list_map_req);
+ list_add_tail(&map_req->list_map_req, &hpb->lh_map_req_free);
+ }
+
+ return 0;
+out_free_req:
+ kfree(hpb->req_info);
+out_free_rsp:
+ kfree(hpb->rsp_info);
+out:
+ return -ENOMEM;
+}
+
+static void ufshpb_lu_constant_init(struct ufshpb_lu *hpb,
+ struct ufshpb_lu_desc *lu_desc,
+ struct ufshpb_func_desc *func_desc)
+{
+ unsigned long long region_size, region_entry_size;
+ unsigned long long last_subregion_size, last_region_size;
+ int entries_per_region;
+
+ region_size = (unsigned long long)
+ SECTOR_SIZE * (0x01 << func_desc->hpb_region_size);
+ region_entry_size =
+ region_size / UFS_LOGICAL_BLOCK_SIZE * HPB_ENTRY_SIZE;
+
+ hpb->subregion_size = (unsigned long long)
+ SECTOR_SIZE * (0x01 << func_desc->hpb_subregion_size);
+ hpb->subregion_entry_size =
+ hpb->subregion_size / UFS_LOGICAL_BLOCK_SIZE * HPB_ENTRY_SIZE;
+
+ hpb->lu_max_active_regions = lu_desc->lu_max_active_hpb_regions;
+ hpb->lru_info.max_active_cnt = lu_desc->lu_max_active_hpb_regions -
+ lu_desc->lu_num_hpb_pinned_regions;
+ /*
+ * relationship across LU,region,subregion,L2P entry:
+ * lu -> region -> sub region -> entry
+ */
+ hpb->lu_num_blocks = lu_desc->lu_logblk_cnt;
+ entries_per_region = region_entry_size / HPB_ENTRY_SIZE;
+ hpb->entries_per_subregion = hpb->subregion_entry_size /
+ HPB_ENTRY_SIZE;
+ hpb->subregions_per_region = region_entry_size /
+ hpb->subregion_entry_size;
+
+ hpb->subregions_in_last_region = hpb->subregions_per_region;
+ hpb->last_subregion_entry_size = hpb->subregion_entry_size;
+ /*
+ * HPB Sub-Regions are equally sized except for the last one which is
+ * smaller if the last HPB Region is not an interger multiple of
+ * bHPBSubRegionSize
+ */
+ last_region_size = (unsigned long long)
+ hpb->lu_num_blocks * UFS_LOGICAL_BLOCK_SIZE %
+ region_size;
+ if (last_region_size) {
+ last_subregion_size = (unsigned long long)
+ last_region_size % hpb->subregion_size;
+ if (last_subregion_size) {
+ hpb->subregions_in_last_region = last_region_size /
+ hpb->subregion_size + 1;
+ hpb->last_subregion_entry_size =
+ last_subregion_size / UFS_LOGICAL_BLOCK_SIZE *
+ HPB_ENTRY_SIZE;
+ } else
+ hpb->subregions_in_last_region = last_region_size /
+ hpb->subregion_size;
+ }
+ /*
+ * 1. regions_per_lu = (lu_num_blocks * 4096) / region_size
+ * = (lu_num_blocks * HPB_ENTRY_SIZE) / region_entry_size
+ * = lu_num_blocks / (region_entry_size / HPB_ENTRY_SIZE)
+ *
+ * 2. regions_per_lu = lu_num_blocks / subregion_entry_size (is trik...)
+ * if HPB_ENTRY_SIZE != subregions_per_region, it is error.
+ */
+ hpb->regions_per_lu =
+ ((unsigned long long)hpb->lu_num_blocks
+ + (region_entry_size / HPB_ENTRY_SIZE) - 1)
+ / (region_entry_size / HPB_ENTRY_SIZE);
+ hpb->subregions_per_lu =
+ ((unsigned long long)hpb->lu_num_blocks
+ + (hpb->subregion_entry_size / HPB_ENTRY_SIZE) - 1)
+ / (hpb->subregion_entry_size / HPB_ENTRY_SIZE);
+
+ /* mempool info */
+ hpb->mpage_bytes = PAGE_SIZE;
+ hpb->mpages_per_subregion = hpb->subregion_entry_size /
+ hpb->mpage_bytes;
+ /* Bitmask Info. */
+ hpb->dwords_per_subregion = hpb->entries_per_subregion / BITS_PER_DWORD;
+ hpb->entries_per_region_shift = ffs(entries_per_region) - 1;
+ hpb->entries_per_region_mask = entries_per_region - 1;
+ hpb->entries_per_subregion_shift = ffs(hpb->entries_per_subregion) - 1;
+ hpb->entries_per_subregion_mask = hpb->entries_per_subregion - 1;
+
+ hpb_info("LU%d Total logical Block Count %d, region count %d\n",
+ hpb->lun, hpb->lu_num_blocks, hpb->regions_per_lu);
+ hpb_info("LU%d Subregions count per lun %d\n",
+ hpb->lun, hpb->subregions_per_lu);
+ hpb_info("LU%d Region size 0x%llx, Region L2P entry size 0x%llx\n",
+ hpb->lun, region_size, region_entry_size);
+ hpb_info("LU%d Subregion size 0x%llx, subregion L2P entry size 0x%x\n",
+ hpb->lun, hpb->subregion_size, hpb->subregion_entry_size);
+ hpb_info("LU%d Last region size 0x%llx, last subregion L2P entry %d\n",
+ hpb->lun, last_region_size, hpb->last_subregion_entry_size);
+
+ hpb_info("LU%d Subregions per region %d\n", hpb->lun,
+ hpb->subregions_per_region);
+ hpb_info("LU%d L2P entries per region %u shift %u mask 0x%x\n",
+ hpb->lun, entries_per_region,
+ hpb->entries_per_region_shift,
+ hpb->entries_per_region_mask);
+ hpb_info("LU%d L2P entries per subregion %u shift %u mask 0x%x\n",
+ hpb->lun, hpb->entries_per_subregion,
+ hpb->entries_per_subregion_shift,
+ hpb->entries_per_subregion_mask);
+ hpb_info("LU%d Count of mpage per subregion %d\n",
+ hpb->lun, hpb->mpages_per_subregion);
+}
+
+static int ufshpb_lu_init(struct ufs_hba *hba, struct ufshpb_lu *hpb,
+ struct ufshpb_func_desc *func_desc,
+ struct ufshpb_lu_desc *lu_desc, int lun)
+{
+ struct ufshpb_region *region_table, *cb;
+ int subregions, subregion_count;
+ int region;
+ bool do_work_handler;
+ int ret = 0;
+ int j;
+
+ hpb->hba = hba;
+ hpb->lun = lun;
+ hpb->debug = 0;
+ hpb->lu_hpb_enable = true;
+
+ ufshpb_lu_constant_init(hpb, lu_desc, func_desc);
+
+ region_table = kcalloc(hpb->regions_per_lu,
+ sizeof(struct ufshpb_region),
+ GFP_KERNEL);
+ if (!region_table)
+ goto out;
+
+ hpb_info("LU%d region table size: %lu bytes\n", lun,
+ (sizeof(struct ufshpb_region) * hpb->regions_per_lu));
+
+ hpb->region_tbl = region_table;
+
+ spin_lock_init(&hpb->hpb_lock);
+ spin_lock_init(&hpb->rsp_list_lock);
+ spin_lock_init(&hpb->req_list_lock);
+
+ /* initialize loacl active region info lru */
+ INIT_LIST_HEAD(&hpb->lru_info.lru);
+ hpb->lru_info.selection_type = LRU;
+
+ INIT_LIST_HEAD(&hpb->lh_subregion_req);
+
+ ret = ufshpb_mctx_mempool_init(hpb, lu_desc->lu_max_active_hpb_regions,
+ hpb->subregions_per_region,
+ hpb->entries_per_subregion,
+ HPB_ENTRY_SIZE);
+ if (ret) {
+ hpb_err("mapping ctx mempool init fail!\n");
+ goto release_mempool;
+ }
+
+ ret = ufshpb_req_mempool_init(hpb, lu_desc->lu_num_hpb_pinned_regions,
+ lu_desc->lu_queue_depth);
+ if (ret) {
+ hpb_err("req&rsp mempool init fail!\n");
+ goto release_mempool;
+ }
+
+ subregions = hpb->subregions_per_lu;
+
+ ufshpb_init_jobs(hpb);
+
+ subregion_count = 0;
+ subregions = hpb->subregions_per_lu;
+ for (region = 0; region < hpb->regions_per_lu; region++) {
+ cb = region_table + region;
+
+ /* Initialize logic unit region information */
+ INIT_LIST_HEAD(&cb->list_region);
+ cb->hit_count = 0;
+ cb->region = region;
+
+ subregion_count = min(subregions,
+ hpb->subregions_per_region);
+
+ ret = ufshpb_alloc_subregion_tbl(hpb, cb, subregion_count);
+ if (ret) {
+ hpb_err("Error while allocating subregion_tbl\n");
+ goto release_subregion_tbl;
+ }
+ ufshpb_init_subregion_tbl(hpb, cb);
+
+ if (ufshpb_is_pinned_region(lu_desc, region)) {
+ hpb_info("Region %d PINNED(%d ~ %d)\n", region,
+ lu_desc->hpb_pinned_region_startidx,
+ lu_desc->lu_hpb_pinned_end_offset);
+ ret = ufshpb_init_pinned_region(hpb, cb);
+ if (ret)
+ goto release_subregion_tbl;
+
+ do_work_handler = true;
+ } else {
+ cb->region_state = REGION_INACTIVE;
+ }
+
+ subregions -= subregion_count;
+ }
+
+ if (subregions != 0) {
+ hpb_err("Error, remaining %d subregions aren't initializd\n",
+ subregions);
+ goto release_subregion_tbl;
+ }
+
+ hpb_info("LU%d subregions info table takes memory %lu bytes\n", lun,
+ sizeof(struct ufshpb_subregion) * hpb->subregions_per_lu);
+
+ if (do_work_handler)
+ schedule_work(&hpb->ufshpb_sync_work);
+
+ /*
+ * Even if creating sysfs failed, ufshpb could run normally.
+ * so we don't deal with error handling
+ */
+ ufshpb_create_sysfs(hba, hpb);
+ return 0;
+
+release_subregion_tbl:
+ for (j = 0; j < region; j++) {
+ cb = region_table + j;
+ ufshpb_destroy_subregion_tbl(hpb, cb);
+ }
+release_mempool:
+ ufshpb_req_mempool_remove(hpb);
+ ufshpb_mctx_mempool_remove(hpb);
+ kfree(region_table);
+out:
+ hpb->lu_hpb_enable = false;
+ return ret;
+}
+
+static int ufshpb_get_hpb_lu_desc(struct ufs_hba *hba,
+ struct ufshpb_lu_desc *lu_desc, int lun)
+{
+ int ret;
+ u8 buf[QUERY_DESC_MAX_SIZE] = { };
+
+ ret = ufshcd_read_unit_desc_param(hba, lun, 0, buf,
+ QUERY_DESC_MAX_SIZE);
+ if (ret) {
+ hpb_err("Read unit desc failed with ret %d\n", ret);
+ return ret;
+ }
+
+ if (buf[UNIT_DESC_PARAM_LU_ENABLE] == 0x02) {
+ lu_desc->lu_hpb_enable = true;
+ hpb_info("LU%d HPB is enabled\n", lun);
+ } else {
+ lu_desc->lu_hpb_enable = false;
+ return 0;
+ }
+
+ lu_desc->lu_queue_depth = buf[UNIT_DESC_PARAM_LU_Q_DEPTH];
+
+ lu_desc->lu_logblk_size = buf[UNIT_DESC_PARAM_LOGICAL_BLK_SIZE];
+ lu_desc->lu_logblk_cnt =
+ SHIFT_BYTE_7((u64)buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT]) |
+ SHIFT_BYTE_6((u64)buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT + 1]) |
+ SHIFT_BYTE_5((u64)buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT + 2]) |
+ SHIFT_BYTE_4((u64)buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT + 3]) |
+ SHIFT_BYTE_3(buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT + 4]) |
+ SHIFT_BYTE_2(buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT + 5]) |
+ SHIFT_BYTE_1(buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT + 6]) |
+ SHIFT_BYTE_0(buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT + 7]);
+
+
+ lu_desc->lu_max_active_hpb_regions =
+ SHIFT_BYTE_1(buf[UNIT_DESC_PARAM_HPB_MAX_ACTIVE_REGIONS]) |
+ SHIFT_BYTE_0(buf[UNIT_DESC_PARAM_HPB_MAX_ACTIVE_REGIONS + 1]);
+ lu_desc->hpb_pinned_region_startidx =
+ SHIFT_BYTE_1(buf[UNIT_DESC_PARAM_HPB_PIN_REGION_START_OFFSET]) |
+ SHIFT_BYTE_0(buf[UNIT_DESC_PARAM_HPB_PIN_REGION_START_OFFSET + 1]);
+ lu_desc->lu_num_hpb_pinned_regions =
+ SHIFT_BYTE_1(buf[UNIT_DESC_PARAM_HPB_NUM_PIN_REGIONS]) |
+ SHIFT_BYTE_0(buf[UNIT_DESC_PARAM_HPB_NUM_PIN_REGIONS + 1]);
+
+ hpb_info("LU%d bLuQueueDepth %d\n", lun, lu_desc->lu_queue_depth);
+ hpb_info("LU%d bLogicalBlockSize %d\n", lun, lu_desc->lu_logblk_size);
+ hpb_info("LU%d qLogicalBlockCount %llu\n", lun, lu_desc->lu_logblk_cnt);
+ hpb_info("LU%d wLUMaxActiveHPBRegions %d\n", lun,
+ lu_desc->lu_max_active_hpb_regions);
+ hpb_info("LU%d wHPBPinnedRegionStartIdx %d\n", lun,
+ lu_desc->hpb_pinned_region_startidx);
+ hpb_info("LU%d wNumHPBPinnedRegions %d\n", lun,
+ lu_desc->lu_num_hpb_pinned_regions);
+
+ if (lu_desc->lu_num_hpb_pinned_regions > 0) {
+ lu_desc->lu_hpb_pinned_end_offset =
+ lu_desc->hpb_pinned_region_startidx +
+ lu_desc->lu_num_hpb_pinned_regions - 1;
+ hpb_info("LU%d PINNED regions: %d - %d\n", lun,
+ lu_desc->hpb_pinned_region_startidx,
+ lu_desc->lu_hpb_pinned_end_offset);
+ } else
+ lu_desc->lu_hpb_pinned_end_offset = -1;
+
+ return 0;
+}
+
+static int ufshpb_read_geo_desc_support(struct ufs_hba *hba,
+ struct ufshpb_func_desc *desc)
+{
+ int err;
+ u8 *geo_buf;
+ size_t buff_len;
+
+ buff_len = max_t(size_t, hba->desc_size.geom_desc,
+ QUERY_DESC_MAX_SIZE);
+ geo_buf = kmalloc(buff_len, GFP_KERNEL);
+ if (!geo_buf) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
+ geo_buf, buff_len);
+ if (err) {
+ hpb_err("%s: Failed reading Device Geometry Desc. err = %d\n",
+ __func__, err);
+ goto out;
+ }
+
+ desc->hpb_region_size = geo_buf[GEOMETRY_DESC_PARAM_HPB_REGION_SIZE];
+ desc->hpb_number_lu = geo_buf[GEOMETRY_DESC_PARAM_HPB_NUMBER_LU];
+ desc->hpb_subregion_size =
+ geo_buf[GEOMETRY_DESC_PARAM_HPB_SUBREGION_SIZE];
+ desc->hpb_device_max_active_regions =
+ (u16) SHIFT_BYTE_1(geo_buf[GEOMETRY_DESC_PARAM_HPB_MAX_ACTIVE_REGIONS]) |
+ (u16) SHIFT_BYTE_0(geo_buf[GEOMETRY_DESC_PARAM_HPB_MAX_ACTIVE_REGIONS + 1]);
+
+ if (desc->hpb_number_lu == 0) {
+ hpb_warn("There is no LU which supports HPB\n");
+ err = -ENODEV;
+ }
+ dev_dbg(hba->dev, "bHPBRegionSize: %u\n", desc->hpb_region_size);
+ dev_dbg(hba->dev, "bHPBNumberLU: %u\n", desc->hpb_number_lu);
+ dev_dbg(hba->dev, "bHPBSubRegionSize: %u\n", desc->hpb_subregion_size);
+ dev_dbg(hba->dev, "wDeviceMaxActiveHPBRegions: %u\n",
+ desc->hpb_device_max_active_regions);
+out:
+ kfree(geo_buf);
+
+ return err;
+}
+
+static void ufshpb_error_handler(struct work_struct *work)
+{
+ struct ufs_hba *hba;
+
+ hba = container_of(work, struct ufs_hba, ufshpb_eh_work);
+ /*
+ * FIXME, so far, we don't have a optimal restoring solution，
+ * in case of fatal error happenningn. current soltion is that
+ * simply kills HPB, in order to not impact normal read.
+ */
+
+ hpb_warn("UFSHPB driver has failed, UFSHCD will run without HPB\n");
+
+ ufshpb_release(hba, HPB_FAILED);
+}
+
+static int ufshpb_exec_init(struct ufs_hba *hba)
+{
+ struct ufshpb_func_desc func_desc;
+ int lun, ret = 1, i;
+ int hpb_dev = 0;
+
+ if (!(hba->dev_info.ufs_features & UFS_FEATURE_SUPPORT_HPB_BIT))
+ goto out;
+
+ ret = ufshpb_read_geo_desc_support(hba, &func_desc);
+ if (ret)
+ goto out;
+
+ ufshpb_init_constant();
+
+ for (lun = 0; lun < hba->dev_info.max_lu_supported; lun++) {
+ struct ufshpb_lu_desc lu_desc;
+
+ hba->ufshpb_lup[lun] = NULL;
+
+ ret = ufshpb_get_hpb_lu_desc(hba, &lu_desc, lun);
+ if (ret)
+ goto out;
+
+ if (lu_desc.lu_hpb_enable == false) {
+ hpb_info("LU%d HPB is disabled\n", lun);
+ continue;
+ }
+
+ hba->ufshpb_lup[lun] =
+ kzalloc(sizeof(struct ufshpb_lu), GFP_KERNEL);
+ if (!hba->ufshpb_lup[lun])
+ goto out_free_mem;
+
+ ret = ufshpb_lu_init(hba, hba->ufshpb_lup[lun],
+ &func_desc, &lu_desc, lun);
+ if (ret) {
+ if (ret == -ENODEV)
+ continue;
+ else
+ goto out_free_mem;
+ }
+ hpb_dev++;
+ hpb_info("LU%d HPB is working now\n", lun);
+ }
+
+ if (hpb_dev == 0) {
+ hpb_warn("No UFSHPB LU to init\n");
+ ret = -ENODEV;
+ goto out_free_mem;
+ }
+
+ INIT_WORK(&hba->ufshpb_eh_work, ufshpb_error_handler);
+ hba->ufshpb_state = HPB_PRESENT;
+
+ return 0;
+
+out_free_mem:
+ for (i = 0; i < hba->dev_info.max_lu_supported; i++)
+ kfree(hba->ufshpb_lup[i]);
+out:
+ hba->ufshpb_state = HPB_NOT_SUPPORTED;
+ return ret;
+}
+
+static void ufshpb_map_loading_trigger(struct ufshpb_lu *hpb,
+ bool dirty, bool only_pinned)
+{
+ int region, subregion;
+ bool do_sync_work = false;
+
+ for (region = 0; region < hpb->regions_per_lu; region++) {
+ struct ufshpb_region *cb;
+ struct ufshpb_subregion *cp;
+
+ cb = hpb->region_tbl + region;
+ if (cb->region_state == REGION_ACTIVE ||
+ cb->region_state == REGION_PINNED) {
+ if ((only_pinned &&
+ cb->region_state == REGION_PINNED) ||
+ !only_pinned) {
+ hpb_dbg(REGION_INFO, hpb,
+ "Add active region %d state %d\n",
+ region, cb->region_state);
+ spin_lock_bh(&hpb->hpb_lock);
+ for (subregion = 0;
+ subregion < cb->subregion_count;
+ subregion++) {
+ cp = cb->subregion_tbl + subregion;
+ ufshpb_add_subregion_to_req_list(hpb,
+ cp);
+ }
+ spin_unlock_bh(&hpb->hpb_lock);
+ do_sync_work = true;
+ }
+
+ if (dirty) {
+ for (subregion = 0;
+ subregion < cb->subregion_count;
+ subregion++) {
+ cp = cb->subregion_tbl + subregion;
+ cp->subregion_state = SUBREGION_DIRTY;
+ }
+ }
+
+ }
+ }
+
+ if (do_sync_work)
+ schedule_work(&hpb->ufshpb_sync_work);
+}
+
+static void ufshpb_purge_active_region(struct ufshpb_lu *hpb)
+{
+ int region, subregion;
+ int state;
+ struct ufshpb_region *cb;
+ struct ufshpb_subregion *cp;
+
+ spin_lock_bh(&hpb->hpb_lock);
+ for (region = 0; region < hpb->regions_per_lu; region++) {
+ cb = hpb->region_tbl + region;
+
+ if (cb->region_state == REGION_INACTIVE) {
+ hpb_dbg(REGION_INFO, hpb,
+ "region %d is inactive\n", region);
+ continue;
+ }
+ if (cb->region_state == REGION_PINNED) {
+ state = SUBREGION_DIRTY;
+ } else if (cb->region_state == REGION_ACTIVE) {
+ state = SUBREGION_UNUSED;
+ ufshpb_cleanup_lru_info(hpb, cb);
+ } else {
+ hpb_notice("Unsupported region state %d\n",
+ cb->region_state);
+ continue;
+ }
+
+ for (subregion = 0; subregion < cb->subregion_count;
+ subregion++) {
+ cp = cb->subregion_tbl + subregion;
+ ufshpb_subregion_mctx_purge(hpb, cp, state);
+ }
+ hpb_dbg(REGION_INFO, hpb, "region %d state %d free_mctx %d",
+ region, state, hpb->free_mctx);
+ }
+ spin_unlock_bh(&hpb->hpb_lock);
+}
+
+static void ufshpb_retrieve_rsp_info(struct ufshpb_lu *hpb)
+{
+ struct ufshpb_rsp_info *rsp_info;
+ unsigned long flags;
+
+ while (1) {
+ spin_lock_irqsave(&hpb->rsp_list_lock, flags);
+ rsp_info = list_first_entry_or_null(&hpb->lh_rsp_info,
+ struct ufshpb_rsp_info,
+ list_rsp_info);
+ if (!rsp_info) {
+ spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
+ break;
+ }
+
+ list_move_tail(&rsp_info->list_rsp_info,
+ &hpb->lh_rsp_info_free);
+
+ spin_unlock_irqrestore(&hpb->rsp_list_lock, flags);
+ }
+}
+
+static void ufshpb_retrieve_req_info(struct ufshpb_lu *hpb)
+{
+ struct ufshpb_req_info *req_info;
+ unsigned long flags;
+
+ while (1) {
+ spin_lock_irqsave(&hpb->req_list_lock, flags);
+ req_info = list_first_entry_or_null(&hpb->lh_req_info,
+ struct ufshpb_req_info,
+ list_req_info);
+ if (!req_info) {
+ spin_unlock_irqrestore(&hpb->req_list_lock, flags);
+ break;
+ }
+
+ list_move_tail(&req_info->list_req_info,
+ &hpb->lh_req_info_free);
+
+ spin_unlock_irqrestore(&hpb->req_list_lock, flags);
+ }
+}
+
+static void ufshpb_reset(struct ufs_hba *hba)
+{
+ struct ufshpb_lu *hpb;
+ int lu;
+
+ for (lu = 0; lu < hba->dev_info.max_lu_supported; lu++) {
+ hpb = hba->ufshpb_lup[lu];
+ if (hpb && hpb->lu_hpb_enable) {
+ hpb_notice("LU%d HPB RESET\n", lu);
+
+ ufshpb_cancel_jobs(hpb);
+ ufshpb_retrieve_rsp_info(hpb);
+ ufshpb_retrieve_req_info(hpb);
+ ufshpb_purge_active_region(hpb);
+ /* FIXME add hpb reset */
+ ufshpb_init_jobs(hpb);
+ }
+ }
+
+ hba->ufshpb_state = HPB_PRESENT;
+}
+
+static void ufshpb_destroy_subregion_tbl(struct ufshpb_lu *hpb,
+ struct ufshpb_region *cb)
+{
+ int subregion;
+
+ if (!cb->subregion_tbl)
+ return;
+
+ for (subregion = 0; subregion < cb->subregion_count; subregion++) {
+ struct ufshpb_subregion *cp;
+
+ cp = cb->subregion_tbl + subregion;
+
+ hpb_info("Subregion %d-> %p state %d mctx-> %p released\n",
+ subregion, cp, cp->subregion_state, cp->mctx);
+
+ cp->subregion_state = SUBREGION_UNUSED;
+ if (cp->mctx)
+ ufshpb_mctx_put(hpb, cp->mctx);
+ }
+
+ kfree(cb->subregion_tbl);
+}
+
+static void ufshpb_destroy_region_tbl(struct ufshpb_lu *hpb)
+{
+ int region;
+
+ for (region = 0; region < hpb->regions_per_lu; region++) {
+ struct ufshpb_region *cb;
+
+ cb = hpb->region_tbl + region;
+ hpb_info("Region %d state %d\n", region,
+ cb->region_state);
+
+ if (cb->region_state == REGION_PINNED ||
+ cb->region_state == REGION_ACTIVE) {
+ cb->region_state = REGION_INACTIVE;
+
+ ufshpb_destroy_subregion_tbl(hpb, cb);
+ }
+ }
+
+ ufshpb_mctx_mempool_remove(hpb);
+ kfree(hpb->region_tbl);
+}
+
+static int ufshpb_sense_data_checkup(struct ufs_hba *hba,
+ struct hpb_sense_data *sense_data)
+{
+ if ((SHIFT_BYTE_1(sense_data->len[0]) |
+ SHIFT_BYTE_0(sense_data->len[1])) != HPB_SENSE_DATA_LEN ||
+ sense_data->desc_type != HPB_DESCRIPTOR_TYPE ||
+ sense_data->additional_len != HPB_ADDITIONAL_LEN ||
+ sense_data->hpb_op == HPB_RSP_NONE ||
+ sense_data->active_region_cnt > MAX_ACTIVE_NUM ||
+ sense_data->inactive_region_cnt > MAX_INACTIVE_NUM) {
+ hpb_err("HPB Sense Data Error\n");
+ hpb_err("Sense Data Length (12h): 0x%x\n",
+ SHIFT_BYTE_1(sense_data->len[0]) |
+ SHIFT_BYTE_0(sense_data->len[1]));
+ hpb_err("Descriptor Type: 0x%x\n",
+ sense_data->desc_type);
+ hpb_err("Additional Length: 0x%x\n",
+ sense_data->additional_len);
+ hpb_err("HPB Operation: 0x%x\n",
+ sense_data->hpb_op);
+ hpb_err("Active HPB Count: 0x%x\n",
+ sense_data->active_region_cnt);
+ hpb_err("Inactive HPB Count: 0x%x\n",
+ sense_data->inactive_region_cnt);
+ return false;
+ }
+ return true;
+}
+
+static void ufshpb_init_workq_fn(struct work_struct *work)
+{
+ struct ufs_hba *hba;
+ struct delayed_work *dwork = to_delayed_work(work);
+ int err;
+#if defined(CONFIG_SCSI_SCAN_ASYNC)
+ unsigned long flags;
+#endif
+
+ hba = container_of(dwork, struct ufs_hba, ufshpb_init_work);
+
+#if defined(CONFIG_SCSI_SCAN_ASYNC)
+ mutex_lock(&hba->host->scan_mutex);
+ spin_lock_irqsave(hba->host->host_lock, flags);
+ if (hba->host->async_scan == 1) {
+ spin_unlock_irqrestore(hba->host->host_lock, flags);
+ mutex_unlock(&hba->host->scan_mutex);
+ hpb_info("Not set scsi-device-info, so re-sched.\n");
+ schedule_delayed_work(&hba->ufshpb_init_work,
+ msecs_to_jiffies(100));
+ return;
+ }
+ spin_unlock_irqrestore(hba->host->host_lock, flags);
+ mutex_unlock(&hba->host->scan_mutex);
+#endif
+
+ if (hba->ufshpb_state == HPB_NEED_INIT) {
+ err = ufshpb_exec_init(hba);
+ if (err) {
+ hpb_warn("%s failed, err %d, UFS runs without HPB\n",
+ __func__, err);
+ }
+ } else if (hba->ufshpb_state == HPB_NEED_RESET) {
+ ufshpb_reset(hba);
+ }
+}
+
+void ufshpb_init(struct ufs_hba *hba)
+{
+ hba->ufshpb_state = HPB_NEED_INIT;
+ INIT_DELAYED_WORK(&hba->ufshpb_init_work, ufshpb_init_workq_fn);
+}
+
+void ufshpb_release(struct ufs_hba *hba, enum UFSHPB_STATE state)
+{
+ int lun;
+ struct ufshpb_lu *hpb;
+
+ hba->ufshpb_state = HPB_FAILED;
+
+ for (lun = 0; lun < hba->dev_info.max_lu_supported; lun++) {
+ hpb = hba->ufshpb_lup[lun];
+
+ if (!hpb)
+ continue;
+
+ if (!hpb->lu_hpb_enable)
+ continue;
+
+ hpb_notice("LU%d HPB will be released\n", lun);
+
+ hba->ufshpb_lup[lun] = NULL;
+
+ hpb->lu_hpb_enable = false;
+
+ ufshpb_cancel_jobs(hpb);
+
+ ufshpb_destroy_region_tbl(hpb);
+
+ ufshpb_req_mempool_remove(hpb);
+
+ kobject_uevent(&hpb->kobj, KOBJ_REMOVE);
+ kobject_del(&hpb->kobj);
+
+ kfree(hpb);
+ }
+
+ if (alloc_mctx != 0)
+ hpb_warn("Warning: mctx not released thoroughly %d",
+ alloc_mctx);
+
+ hba->ufshpb_state = state;
+}
+
+void ufshpb_rsp_handler(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
+{
+ struct ufshpb_lu *hpb;
+ struct hpb_sense_data *sense_data;
+ struct ufshpb_rsp_info *rsp_info;
+ int data_seg_len;
+
+ if (hba->ufshpb_state != HPB_PRESENT)
+ return;
+
+ BUG_ON(!lrbp);
+ BUG_ON(!lrbp->ucd_rsp_ptr);
+
+ /* Well Known LU doesn't support HPB */
+ if (lrbp->lun >= hba->dev_info.max_lu_supported)
+ return;
+
+ data_seg_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2)
+ & MASK_RSP_UPIU_DATA_SEG_LEN;
+ /* FIXME Add Device informaiton checkup bit 1: HPB_UPDATE_ALERT = 1 */
+ if (!data_seg_len) {
+ bool do_tasklet = false;
+
+ hpb = hba->ufshpb_lup[lrbp->lun];
+ if (!hpb)
+ return;
+ /*
+ * No recommmended region in response, but we still have
+ * recommmended region to enactive in list lh_rsp_info.
+ */
+ spin_lock(&hpb->rsp_list_lock);
+ do_tasklet = !list_empty(&hpb->lh_rsp_info);
+ spin_unlock(&hpb->rsp_list_lock);
+
+ if (do_tasklet)
+ schedule_work(&hpb->ufshpb_rsp_work);
+ return;
+ }
+
+ sense_data = ufshpb_get_sense_data(lrbp);
+
+ BUG_ON(!sense_data);
+
+ if (!ufshpb_sense_data_checkup(hba, sense_data))
+ return;
+
+ hpb = hba->ufshpb_lup[lrbp->lun];
+ if (!hpb) {
+ hpb_warn("%s LU%d didn't allocated HPB\n", __func__,
+ lrbp->lun);
+ return;
+ }
+
+ hpb_dbg(REGION_INFO, hpb,
+ "RSP UPIU for LU%d: HPB OP %d, Data Len 0x%x\n",
+ sense_data->lun, sense_data->hpb_op, data_seg_len);
+
+ atomic64_inc(&hpb->status.rb_noti_cnt);
+
+ if (!hpb->lu_hpb_enable) {
+ hpb_warn("LU%d HPB has been disabled.\n", lrbp->lun);
+ return;
+ }
+
+ rsp_info = ufshpb_rsp_info_get(hpb);
+ if (!rsp_info) {
+ hpb_warn("%s:%d There is no free rsp_info\n",
+ __func__, __LINE__);
+ return;
+ }
+
+ switch (sense_data->hpb_op) {
+ case HPB_REQ_REGION_UPDATE:
+ WARN_ON(data_seg_len != HPB_DATA_SEG_LEN);
+ ufshpb_rsp_info_fill(hpb, rsp_info, sense_data);
+
+ hpb_trace(hpb, "%s: #ACT %d, #INACT %d", DRIVER_NAME,
+ rsp_info->active_cnt, rsp_info->inactive_cnt);
+
+ spin_lock(&hpb->rsp_list_lock);
+ list_add_tail(&rsp_info->list_rsp_info, &hpb->lh_rsp_info);
+ spin_unlock(&hpb->rsp_list_lock);
+
+ schedule_work(&hpb->ufshpb_rsp_work);
+ break;
+
+ default:
+ hpb_warn("HPB Operation %d is not supported\n",
+ sense_data->hpb_op);
+ break;
+ }
+}
+
+void ufshpb_retrieve_l2p_entry(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
+{
+ struct request *rq;
+ struct ufshpb_lu *hpb;
+ struct ufshpb_region *cb;
+ struct ufshpb_subregion *cp;
+ unsigned long long ppn = 0;
+ unsigned long long rq_pos;
+ unsigned int rq_sectors;
+ unsigned int lpn;
+ int region, subregion, subregion_offset;
+ int do_req_workq = 0;
+ int blk_cnt = 1;
+
+ /* WKLU doesn't support HPB */
+ if (!lrbp || lrbp->lun >= hba->dev_info.max_lu_supported)
+ return;
+
+ hpb = hba->ufshpb_lup[lrbp->lun];
+ if (!hpb)
+ return;
+
+ /* If HPB has been disabled or not enabled yet */
+ if (!hpb->lu_hpb_enable || hpb->force_disable)
+ return;
+ /*
+ * Convert sector address(in 512B unit) to block
+ * address (in 4KB unit), since minimum Logical
+ * block size supported by HPB is 4KB.
+ */
+ rq = lrbp->cmd->request;
+ rq_pos = blk_rq_pos(rq);
+ rq_sectors = blk_rq_sectors(rq);
+ lpn = rq_pos / SECTORS_PER_BLOCK;
+
+ /* Get region, subregion and offset according to page address */
+ ufshpb_get_pos_from_lpn(hpb, lpn, ®ion,
+ &subregion, &subregion_offset);
+ cb = hpb->region_tbl + region;
+
+ /*
+ * For the host control mode, we need to watch out
+ * for the write/discard requst.
+ */
+ if (hba->dev_info.hpb_control_mode == HPB_HOST_CTRL_MODE &&
+ ufshpb_is_write_discard_lrbp(lrbp)) {
+ spin_lock_bh(&hpb->hpb_lock);
+
+ if (cb->region_state == REGION_INACTIVE) {
+ spin_unlock_bh(&hpb->hpb_lock);
+ return;
+ }
+ ufshpb_l2p_entry_dirty_set(hpb, lrbp, region,
+ subregion, subregion_offset);
+ spin_unlock_bh(&hpb->hpb_lock);
+ return;
+ }
+
+ /* Check if it is the read cmd */
+ if (!ufshpb_is_read_lrbp(lrbp))
+ return;
+
+ cp = cb->subregion_tbl + subregion;
+
+ spin_lock_bh(&hpb->hpb_lock);
+
+ if (!ufshpb_subregion_is_clean(hpb, cp)) {
+ if (hba->dev_info.hpb_control_mode == HPB_HOST_CTRL_MODE &&
+ cp->subregion_state != SUBREGION_REGISTERED &&
+ cp->subregion_state != SUBREGION_ISSUED) {
+ /*
+ * If this subregion is dirty or inactive, meanwhile,
+ * it hasn't been added to activate_list, for the
+ * host control mode, we need to increase its
+ * read_counter and wake up req_workq
+ */
+ if (atomic_read(&cp->read_counter) < 0xffffffff)
+ atomic_inc(&cp->read_counter);
+
+ if (atomic_read(&cp->read_counter) >=
+ HPB_READ_THREASHOLD) {
+ if (!ufshpb_subregion_register(hpb, cp))
+ do_req_workq = 1;
+ }
+ }
+ spin_unlock_bh(&hpb->hpb_lock);
+ if (do_req_workq) {
+ hpb_dbg(SCHEDULE_INFO, hpb,
+ "Schedule req_workq to activate RG_SRG %d:%d\n",
+ cp->region, cp->subregion);
+ schedule_work(&hpb->ufshpb_req_work);
+ }
+ return;
+ }
+
+ if (rq_sectors > SECTORS_PER_BLOCK) {
+ /*
+ * This version (HPB 1.0) only supports 4KB logical
+ * block size storage devices.
+ */
+ spin_unlock_bh(&hpb->hpb_lock);
+ return;
+ }
+
+ if (ufshpb_l2p_entry_dirty_check(hpb, cp, subregion_offset)) {
+ atomic64_inc(&hpb->status.entry_dirty_miss);
+ hpb_dbg(REGION_INFO, hpb,
+ "0x%llx + %u HPB PPN Dirty %d - %d\n",
+ rq_pos, rq_sectors, region, subregion);
+ spin_unlock_bh(&hpb->hpb_lock);
+ return;
+ }
+
+ ppn = ufshpb_get_ppn(cp->mctx, subregion_offset);
+
+ if (cb->subregion_count == 1)
+ ufshpb_hit_lru_info(&hpb->lru_info, cb);
+
+ spin_unlock_bh(&hpb->hpb_lock);
+
+ if (rq_sectors != SECTORS_PER_BLOCK) {
+ int remainder;
+
+ remainder = rq_sectors % SECTORS_PER_BLOCK;
+ blk_cnt = rq_sectors / SECTORS_PER_BLOCK;
+ if (remainder)
+ blk_cnt += 1;
+ hpb_dbg(NORMAL_INFO, hpb,
+ "Read %d blks (%d sects) from 0x%llx, %d: %d\n",
+ blk_cnt, rq_sectors, rq_pos, region, subregion);
+ }
+
+ ufshpb_prepare_ppn(hpb, lrbp, ppn, blk_cnt);
+ hpb_trace(hpb, "%s: %llu + %u HPB READ %d - %d", DRIVER_NAME, rq_pos,
+ rq_sectors, region, subregion);
+ hpb_dbg(NORMAL_INFO, hpb,
+ "PPN:0x%llx, LA:0x%llx + %u HPB READ from RG %d - SRG %d",
+ ppn, rq_pos, rq_sectors, region, subregion);
+ atomic64_inc(&hpb->status.hit);
+}
+
+static void ufshpb_sysfs_stat_init(struct ufshpb_lu *hpb)
+{
+ atomic64_set(&hpb->status.hit, 0);
+ atomic64_set(&hpb->status.miss, 0);
+ atomic64_set(&hpb->status.region_miss, 0);
+ atomic64_set(&hpb->status.subregion_miss, 0);
+ atomic64_set(&hpb->status.entry_dirty_miss, 0);
+ atomic64_set(&hpb->status.rb_noti_cnt, 0);
+ atomic64_set(&hpb->status.map_req_cnt, 0);
+ atomic64_set(&hpb->status.region_evict, 0);
+ atomic64_set(&hpb->status.region_add, 0);
+ atomic64_set(&hpb->status.rb_fail, 0);
+}
+
+static ssize_t ufshpb_sysfs_debug_release_store(struct ufshpb_lu *hpb,
+ const char *buf, size_t count)
+{
+ unsigned long value;
+
+ hpb_info("Start HPB Release FUNC");
+
+ if (kstrtoul(buf, 0, &value)) {
+ hpb_err("kstrtoul error");
+ return -EINVAL;
+ }
+
+ if (value == 0) {
+ hpb_info("LU%d HPB will be killed manually", hpb->lun);
+ goto kill_hpb;
+ } else {
+ hpb_info("Unrecognize value inputted %lu", value);
+ }
+
+ return count;
+kill_hpb:
+ hpb->hba->ufshpb_state = HPB_FAILED;
+ schedule_work(&hpb->hba->ufshpb_eh_work);
+ return count;
+}
+
+static ssize_t ufshpb_sysfs_info_lba_store(struct ufshpb_lu *hpb,
+ const char *buf, size_t count)
+{
+ unsigned long long ppn;
+ unsigned long value;
+ unsigned int lpn;
+ int region, subregion, subregion_offset;
+ struct ufshpb_region *cb;
+ struct ufshpb_subregion *cp;
+ int dirty;
+
+ if (kstrtoul(buf, 0, &value)) {
+ hpb_err("kstrtoul error\n");
+ return -EINVAL;
+ }
+
+ if (value > hpb->lu_num_blocks * SECTORS_PER_BLOCK) {
+ hpb_err("Error: value %lu > lu_num_blocks %d\n",
+ value, hpb->lu_num_blocks);
+ return -EINVAL;
+ }
+ lpn = value / SECTORS_PER_BLOCK;
+
+ ufshpb_get_pos_from_lpn(hpb, lpn, ®ion, &subregion,
+ &subregion_offset);
+
+ cb = hpb->region_tbl + region;
+ cp = cb->subregion_tbl + subregion;
+
+ if (cb->region_state != REGION_INACTIVE) {
+ ppn = ufshpb_get_ppn(cp->mctx, subregion_offset);
+ spin_lock_bh(&hpb->hpb_lock);
+ dirty = ufshpb_l2p_entry_dirty_check(hpb, cp, subregion_offset);
+ spin_unlock_bh(&hpb->hpb_lock);
+ } else {
+ ppn = 0;
+ dirty = -1;
+ }
+
+ hpb_info("sector %lu region %d state %d subregion %d state %d",
+ value, region, cb->region_state, subregion,
+ cp->subregion_state);
+ hpb_info("sector %lu lpn %u ppn %llx dirty %d",
+ value, lpn, ppn, dirty);
+ return count;
+}
+
+static ssize_t ufshpb_sysfs_map_req_show(struct ufshpb_lu *hpb, char *buf)
+{
+ long long rb_noti_cnt, map_req_cnt;
+
+ rb_noti_cnt = atomic64_read(&hpb->status.rb_noti_cnt);
+ map_req_cnt = atomic64_read(&hpb->status.map_req_cnt);
+
+ return snprintf(buf, PAGE_SIZE,
+ "Recommend RSP count %lld HPB Buffer Read count %lld\n",
+ rb_noti_cnt, map_req_cnt);
+}
+
+static ssize_t ufshpb_sysfs_count_reset_store(struct ufshpb_lu *hpb,
+ const char *buf, size_t count)
+{
+ unsigned long debug;
+
+ if (kstrtoul(buf, 0, &debug))
+ return -EINVAL;
+
+ ufshpb_sysfs_stat_init(hpb);
+
+ return count;
+}
+
+static ssize_t ufshpb_sysfs_add_evict_show(struct ufshpb_lu *hpb, char *buf)
+{
+ long long add, evict;
+
+ add = atomic64_read(&hpb->status.region_add);
+ evict = atomic64_read(&hpb->status.region_evict);
+
+ return snprintf(buf, PAGE_SIZE, "Add %lld, Evict %lld\n", add, evict);
+}
+
+static ssize_t ufshpb_sysfs_hit_show(struct ufshpb_lu *hpb, char *buf)
+{
+ long long hit;
+
+ hit = atomic64_read(&hpb->status.hit);
+
+ return snprintf(buf, PAGE_SIZE, "LU%d HPB Total hit: %lld\n",
+ hpb->lun, hit);
+}
+
+static ssize_t ufshpb_sysfs_miss_show(struct ufshpb_lu *hpb, char *buf)
+{
+ long long region_miss, subregion_miss, entry_dirty_miss, rb_fail;
+ int ret = 0, count = 0;
+
+ region_miss = atomic64_read(&hpb->status.region_miss);
+ subregion_miss = atomic64_read(&hpb->status.subregion_miss);
+ entry_dirty_miss = atomic64_read(&hpb->status.entry_dirty_miss);
+ rb_fail = atomic64_read(&hpb->status.rb_fail);
+
+ ret = sprintf(buf + count, "Total entries missed: %lld\n",
+ region_miss + subregion_miss + entry_dirty_miss);
+ count += ret;
+
+ ret = sprintf(buf + count, "Region: %lld\n", region_miss);
+ count += ret;
+
+ ret = sprintf(buf + count, "subregion: %lld, entry dirty: %lld\n",
+ subregion_miss, entry_dirty_miss);
+ count += ret;
+
+ ret = sprintf(buf + count, "HPB Read Buffer CMD failure: %lld\n",
+ rb_fail);
+
+ return (count + ret);
+}
+
+static ssize_t ufshpb_sysfs_active_region_show(struct ufshpb_lu *hpb, char *buf)
+{
+ int ret = 0, count = 0, region;
+
+ ret = sprintf(buf, " ACTIVE Region List:\n");
+ count = ret;
+
+ for (region = 0; region < hpb->regions_per_lu; region++) {
+ if (hpb->region_tbl[region].region_state == REGION_ACTIVE ||
+ hpb->region_tbl[region].region_state == REGION_PINNED) {
+ ret = sprintf(buf + count, "%d ", region);
+ count += ret;
+ }
+ }
+
+ ret = sprintf(buf + count, "\n");
+ count += ret;
+
+ return count;
+}
+
+static ssize_t ufshpb_sysfs_subregion_status_show(struct ufshpb_lu *hpb,
+ char *buf)
+{
+ int ret = 0, count = 0, region, sub;
+ struct ufshpb_region *cb;
+ int state;
+
+ ret = sprintf(buf, "Clean Subregion List:\n");
+ count = ret;
+
+ for (region = 0; region < hpb->regions_per_lu; region++) {
+ cb = hpb->region_tbl + region;
+ for (sub = 0;
+ sub < cb->subregion_count;
+ sub++) {
+ state = cb->subregion_tbl[sub].subregion_state;
+ if (state == SUBREGION_CLEAN) {
+ ret = sprintf(buf + count, "%d:%d ",
+ region, sub);
+ count += ret;
+ }
+ }
+ }
+
+ ret = sprintf(buf + count, "\n");
+ count += ret;
+
+ return count;
+}
+
+static ssize_t ufshpb_sysfs_debug_store(struct ufshpb_lu *hpb,
+ const char *buf, size_t count)
+{
+ unsigned long debug;
+
+ if (kstrtoul(buf, 0, &debug))
+ return -EINVAL;
+
+ if (debug >= 0)
+ hpb->debug = debug;
+ else
+ hpb->debug = 0;
+
+ return count;
+}
+
+static ssize_t ufshpb_sysfs_debug_show(struct ufshpb_lu *hpb, char *buf)
+{
+
+ return snprintf(buf, PAGE_SIZE, "%d\n", hpb->debug);
+}
+
+static ssize_t ufshpb_sysfs_map_loading_store(struct ufshpb_lu *hpb,
+ const char *buf, size_t count)
+{
+ unsigned long value;
+
+ if (kstrtoul(buf, 0, &value))
+ return -EINVAL;
+
+ if (value > 1)
+ return -EINVAL;
+
+ if (value == 1)
+ ufshpb_map_loading_trigger(hpb, false, false);
+
+ return count;
+}
+
+static ssize_t ufshpb_sysfs_map_disable_show(struct ufshpb_lu *hpb, char *buf)
+{
+ return snprintf(buf, PAGE_SIZE,
+ ">> force_map_req_disable: %d\n",
+ hpb->force_map_req_disable);
+}
+
+static ssize_t ufshpb_sysfs_map_disable_store(struct ufshpb_lu *hpb,
+ const char *buf, size_t count)
+{
+ unsigned long value;
+
+ if (kstrtoul(buf, 0, &value))
+ return -EINVAL;
+
+ if (value > 1)
+ value = 1;
+
+ if (value == 1)
+ hpb->force_map_req_disable = true;
+ else if (value == 0)
+ hpb->force_map_req_disable = false;
+ else
+ hpb_err("Error value: %lu", value);
+
+ return count;
+}
+
+static ssize_t ufshpb_sysfs_disable_show(struct ufshpb_lu *hpb, char *buf)
+{
+ return snprintf(buf, PAGE_SIZE,
+ "LU%d HPB force_disable: %d\n",
+ hpb->lun, hpb->force_disable);
+}
+
+static ssize_t ufshpb_sysfs_disable_store(struct ufshpb_lu *hpb,
+ const char *buf, size_t count)
+{
+ unsigned long value;
+
+ if (kstrtoul(buf, 0, &value))
+ return -EINVAL;
+
+ if (value > 1)
+ value = 1;
+
+ if (value == 1)
+ hpb->force_disable = true;
+ else if (value == 0)
+ hpb->force_disable = false;
+ else
+ hpb_err("Error value: %lu\n", value);
+
+ return count;
+}
+
+static int global_region;
+
+static inline bool is_region_active(struct ufshpb_lu *hpb, int region)
+{
+ if (hpb->region_tbl[region].region_state == REGION_ACTIVE ||
+ hpb->region_tbl[region].region_state == REGION_PINNED)
+ return true;
+
+ return false;
+}
+
+static ssize_t ufshpb_sysfs_active_group_store(struct ufshpb_lu *hpb,
+ const char *buf, size_t count)
+{
+ unsigned long block;
+ int region;
+
+ if (kstrtoul(buf, 0, &block))
+ return -EINVAL;
+
+ region = block >> hpb->entries_per_region_shift;
+ if (region >= hpb->regions_per_lu) {
+ hpb_err("Region %d exceeds max %d", region,
+ hpb->regions_per_lu);
+ region = hpb->regions_per_lu - 1;
+ }
+
+ global_region = region;
+
+ hpb_info("block %lu region %d active %d",
+ block, region, is_region_active(hpb, region));
+
+ return count;
+}
+
+static ssize_t ufshpb_sysfs_active_group_show(struct ufshpb_lu *hpb, char *buf)
+{
+
+ return snprintf(buf, PAGE_SIZE,
+ "Region %d: %d\n", global_region,
+ is_region_active(hpb, global_region));
+}
+
+static struct ufshpb_sysfs_entry ufshpb_sysfs_entries[] = {
+ __ATTR(is_active_group, 0644,
+ ufshpb_sysfs_active_group_show, ufshpb_sysfs_active_group_store),
+ __ATTR(bypass_hpb, 0644,
+ ufshpb_sysfs_disable_show, ufshpb_sysfs_disable_store),
+ __ATTR(map_cmd_disable, 0644,
+ ufshpb_sysfs_map_disable_show, ufshpb_sysfs_map_disable_store),
+ __ATTR(sync_active_region, 0200, NULL, ufshpb_sysfs_map_loading_store),
+ __ATTR(debug, 0644, ufshpb_sysfs_debug_show,
+ ufshpb_sysfs_debug_store),
+ __ATTR(active_subregion, 0444, ufshpb_sysfs_subregion_status_show,
+ NULL),
+ __ATTR(total_hit_count, 0444, ufshpb_sysfs_hit_show, NULL),
+ __ATTR(miss_count, 0444, ufshpb_sysfs_miss_show, NULL),
+ __ATTR(active_region, 0444, ufshpb_sysfs_active_region_show, NULL),
+ __ATTR(region_add_evict_count, 0444, ufshpb_sysfs_add_evict_show, NULL),
+ __ATTR(count_reset, 0200, NULL, ufshpb_sysfs_count_reset_store),
+ __ATTR(rsq_req_count, 0444, ufshpb_sysfs_map_req_show, NULL),
+ __ATTR(get_info_from_lba, 0200, NULL, ufshpb_sysfs_info_lba_store),
+ __ATTR(kill_hpb, 0200, NULL, ufshpb_sysfs_debug_release_store),
+ __ATTR_NULL
+};
+
+static ssize_t ufshpb_attr_show(struct kobject *kobj, struct attribute *attr,
+ char *page)
+{
+ struct ufshpb_sysfs_entry *entry;
+ struct ufshpb_lu *hpb;
+ ssize_t error;
+
+ entry = container_of(attr, struct ufshpb_sysfs_entry, attr);
+ hpb = container_of(kobj, struct ufshpb_lu, kobj);
+
+ if (!entry->show)
+ return -EIO;
+
+ mutex_lock(&hpb->sysfs_lock);
+ error = entry->show(hpb, page);
+ mutex_unlock(&hpb->sysfs_lock);
+ return error;
+}
+
+static ssize_t ufshpb_attr_store(struct kobject *kobj, struct attribute *attr,
+ const char *page, size_t length)
+{
+ struct ufshpb_sysfs_entry *entry;
+ struct ufshpb_lu *hpb;
+ ssize_t error;
+
+ entry = container_of(attr, struct ufshpb_sysfs_entry, attr);
+ hpb = container_of(kobj, struct ufshpb_lu, kobj);
+
+ if (!entry->store)
+ return -EIO;
+
+ mutex_lock(&hpb->sysfs_lock);
+ error = entry->store(hpb, page, length);
+ mutex_unlock(&hpb->sysfs_lock);
+ return error;
+}
+
+static const struct sysfs_ops ufshpb_sysfs_ops = {
+ .show = ufshpb_attr_show,
+ .store = ufshpb_attr_store,
+};
+
+static struct kobj_type ufshpb_ktype = {
+ .sysfs_ops = &ufshpb_sysfs_ops,
+ .release = NULL,
+};
+
+static int ufshpb_create_sysfs(struct ufs_hba *hba, struct ufshpb_lu *hpb)
+{
+ struct device *dev = hba->dev;
+ struct ufshpb_sysfs_entry *entry;
+ int err;
+
+ hpb->sysfs_entries = ufshpb_sysfs_entries;
+
+ ufshpb_sysfs_stat_init(hpb);
+
+ kobject_init(&hpb->kobj, &ufshpb_ktype);
+ mutex_init(&hpb->sysfs_lock);
+
+ err = kobject_add(&hpb->kobj, kobject_get(&dev->kobj),
+ "ufshpb_lu%d", hpb->lun);
+ if (!err) {
+ for (entry = hpb->sysfs_entries; entry->attr.name != NULL;
+ entry++) {
+ if (sysfs_create_file(&hpb->kobj, &entry->attr))
+ break;
+ }
+ kobject_uevent(&hpb->kobj, KOBJ_ADD);
+ }
+
+ return err;
+}
new file mode 100644
@@ -0,0 +1,421 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Universal Flash Storage Host Performance Booster
+ *
+ * Copyright (C) 2017-2018 Samsung Electronics Co., Ltd.
+ *
+ * Authors:
+ * Yongmyung Lee <ymhungry.lee@samsung.com>
+ * Jinyoung Choi <j-young.choi@samsung.com>
+ */
+
+#ifndef _UFSHPB_H_
+#define _UFSHPB_H_
+
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <linux/blkdev.h>
+
+#include "ufshcd.h"
+
+#define UFSHPB_VER 0x0100
+#define DRIVER_NAME "UFSHPB"
+#define hpb_warn(fmt, ...) \
+ pr_warn("%s:" fmt, DRIVER_NAME, ##__VA_ARGS__)
+#define hpb_notice(fmt, ...) \
+ pr_notice("%s:" fmt, DRIVER_NAME, ##__VA_ARGS__)
+#define hpb_info(fmt, ...) \
+ pr_info("%s:" fmt, DRIVER_NAME, ##__VA_ARGS__)
+#define hpb_err(fmt, ...) \
+ pr_err("%s: %s:" fmt, __func__, DRIVER_NAME, \
+ ##__VA_ARGS__)
+#define hpb_dbg(level, hpb, fmt, ...) \
+do { \
+ if (hpb == NULL) \
+ break; \
+ if (hpb->debug >= HPB_DBG_MAX || hpb->debug == level) \
+ pr_notice("%s: %s():" fmt, \
+ DRIVER_NAME, __func__, ##__VA_ARGS__); \
+} while (0)
+
+/* HPB control mode */
+#define HPB_HOST_CTRL_MODE 0x00
+#define HPB_DEV_CTRL_MODE 0x01
+
+/*
+ * The default read count threashold before activating subregion.
+ * Used in HPB host control mode.
+ */
+#define HPB_READ_THREASHOLD 10
+
+/* Constant value*/
+#define UFS_LOGICAL_BLOCK_SIZE 4096
+#define SECTORS_PER_BLOCK (UFS_LOGICAL_BLOCK_SIZE / SECTOR_SIZE)
+#define BITS_PER_DWORD 32
+#define MAX_ACTIVE_NUM 2
+#define MAX_INACTIVE_NUM 2
+
+#define HPB_ENTRY_SIZE 0x08
+#define HPB_ENTREIS_PER_OS_PAGE (PAGE_SIZE / HPB_ENTRY_SIZE)
+
+/* Description */
+#define UFS_FEATURE_SUPPORT_HPB_BIT 0x80
+
+#define PER_ACTIVE_INFO_BYTES 4
+#define PER_INACTIVE_INFO_BYTES 2
+
+/* UFS HPB OPCODE */
+#define UFSHPB_READ 0xF8
+#define UFSHPB_READ_BUFFER 0xF9
+#define UFSHPB_WRITE_BUFFER 0xFA
+
+#define HPB_DATA_SEG_LEN 0x14
+#define HPB_SENSE_DATA_LEN 0x12
+#define HPB_DESCRIPTOR_TYPE 0x80
+#define HPB_ADDITIONAL_LEN 0x10
+
+/* BYTE SHIFT */
+#define ZERO_BYTE_SHIFT 0
+#define ONE_BYTE_SHIFT 8
+#define TWO_BYTE_SHIFT 16
+#define THREE_BYTE_SHIFT 24
+#define FOUR_BYTE_SHIFT 32
+#define FIVE_BYTE_SHIFT 40
+#define SIX_BYTE_SHIFT 48
+#define SEVEN_BYTE_SHIFT 56
+
+#define SHIFT_BYTE_0(num) ((num) << ZERO_BYTE_SHIFT)
+#define SHIFT_BYTE_1(num) ((num) << ONE_BYTE_SHIFT)
+#define SHIFT_BYTE_2(num) ((num) << TWO_BYTE_SHIFT)
+#define SHIFT_BYTE_3(num) ((num) << THREE_BYTE_SHIFT)
+#define SHIFT_BYTE_4(num) ((num) << FOUR_BYTE_SHIFT)
+#define SHIFT_BYTE_5(num) ((num) << FIVE_BYTE_SHIFT)
+#define SHIFT_BYTE_6(num) ((num) << SIX_BYTE_SHIFT)
+#define SHIFT_BYTE_7(num) ((num) << SEVEN_BYTE_SHIFT)
+
+#define GET_BYTE_0(num) (((num) >> ZERO_BYTE_SHIFT) & 0xff)
+#define GET_BYTE_1(num) (((num) >> ONE_BYTE_SHIFT) & 0xff)
+#define GET_BYTE_2(num) (((num) >> TWO_BYTE_SHIFT) & 0xff)
+#define GET_BYTE_3(num) (((num) >> THREE_BYTE_SHIFT) & 0xff)
+#define GET_BYTE_4(num) (((num) >> FOUR_BYTE_SHIFT) & 0xff)
+#define GET_BYTE_5(num) (((num) >> FIVE_BYTE_SHIFT) & 0xff)
+#define GET_BYTE_6(num) (((num) >> SIX_BYTE_SHIFT) & 0xff)
+#define GET_BYTE_7(num) (((num) >> SEVEN_BYTE_SHIFT) & 0xff)
+
+//#define REGION_UNIT_SIZE(bit_offset) (0x01 << (bit_offset))
+#define MAX_BVEC_SIZE 128
+
+enum UFSHPB_STATE {
+ HPB_PRESENT = 1,
+ HPB_NOT_SUPPORTED,
+ HPB_FAILED,
+ HPB_NEED_INIT,
+ HPB_NEED_RESET,
+};
+
+enum HPBREGION_STATE {
+ REGION_INACTIVE,
+ REGION_ACTIVE,
+ REGION_PINNED,
+};
+
+enum HPBSUBREGION_STATE {
+ SUBREGION_UNUSED = 1,
+ SUBREGION_DIRTY,
+ SUBREGION_REGISTERED,
+ SUBREGION_ISSUED,
+ SUBREGION_CLEAN,
+};
+
+enum HPB_REQUEST_TYPE {
+ HPB_REGION_DEACTIVATE = 1,
+ HPB_SUBREGION_ACTIVATE,
+};
+
+/* Response UPIU types */
+enum HPB_SEMSE_DATA_OPERATION {
+ HPB_RSP_NONE,
+ HPB_REQ_REGION_UPDATE,
+ HPB_RESET_REGION_INFO,
+};
+
+enum HPB_DEBUG_LEVEL {
+ FAILURE_INFO = 1,
+ REGION_INFO,
+ SCHEDULE_INFO,
+ NORMAL_INFO,
+ HPB_DBG_MAX,
+};
+
+struct ufshpb_func_desc {
+ /*** Geometry Descriptor ***/
+ /* 48h bHPBRegionSize (UNIT: 512B) */
+ u8 hpb_region_size;
+ /* 49h bHPBNumberLU */
+ u8 hpb_number_lu;
+ /* 4Ah bHPBSubRegionSize */
+ u8 hpb_subregion_size;
+ /* 4B:4Ch wDeviceMaxActiveHPBRegions */
+ u16 hpb_device_max_active_regions;
+};
+
+struct ufshpb_lu_desc {
+ /*** Unit Descriptor ****/
+ /* 03h bLUEnable */
+ int lu_enable;
+ /* 06h lu queue depth info*/
+ int lu_queue_depth;
+ /* 0Ah bLogicalBlockSize. default 0x0C = 4KB */
+ int lu_logblk_size;
+ /* 0Bh qLogicalBlockCount, total number of addressable logical blocks */
+ u64 lu_logblk_cnt;
+
+ /* 23h:24h wLUMaxActiveHPBRegions */
+ u16 lu_max_active_hpb_regions;
+ /* 25h:26h wHPBPinnedRegionStartIdx */
+ u16 hpb_pinned_region_startidx;
+ /* 27h:28h wNumHPBPinnedRegions */
+ u16 lu_num_hpb_pinned_regions;
+
+
+ /* if 03h value is 02h, hpb_enable is set. */
+ bool lu_hpb_enable;
+
+ int lu_hpb_pinned_end_offset;
+};
+
+struct ufshpb_rsp_active_list {
+ /* Active HPB Region 0/1 */
+ u16 region[MAX_ACTIVE_NUM];
+ /* HPB Sub-Region of Active HPB Region 0/1 */
+ u16 subregion[MAX_ACTIVE_NUM];
+};
+
+struct ufshpb_rsp_inactive_list {
+ /* Inactive HPB Region 0/1 */
+ u16 region[MAX_INACTIVE_NUM];
+};
+
+/*
+ * HPB response information
+ */
+struct ufshpb_rsp_info {
+ int type;
+ int active_cnt;
+ int inactive_cnt;
+ struct ufshpb_rsp_active_list active_list;
+ struct ufshpb_rsp_inactive_list inactive_list;
+
+ __u64 rsp_start;
+ __u64 rsp_tasklet_enter;
+
+ struct list_head list_rsp_info;
+};
+
+/*
+ * HPB request info to activate/deactivate subregion/region
+ */
+struct ufshpb_req_info {
+ enum HPB_REQUEST_TYPE type;
+ int region;
+ int subregion;
+
+ __u64 req_start_t;
+ __u64 workq_enter_t;
+
+ struct list_head list_req_info;
+};
+
+/*
+ * HPB Sense Data frame in RESPONSE UPIU
+ */
+struct hpb_sense_data {
+ u8 len[2];
+ u8 desc_type;
+ u8 additional_len;
+ u8 hpb_op;
+ u8 lun;
+ u8 active_region_cnt;
+ u8 inactive_region_cnt;
+ u8 active_field[8];
+ u8 inactive_field[4];
+};
+
+struct ufshpb_map_ctx {
+ struct page **m_page;
+ unsigned int *ppn_dirty;
+ unsigned int ppn_dirty_counter;
+
+ struct list_head list_table;
+};
+
+struct ufshpb_subregion {
+ struct ufshpb_map_ctx *mctx;
+ enum HPBSUBREGION_STATE subregion_state;
+ int region;
+ int subregion;
+ /* Counter of read hit on this subregion */
+ atomic_t read_counter;
+
+ struct list_head list_subregion;
+};
+
+struct ufshpb_region {
+ struct ufshpb_subregion *subregion_tbl;
+ enum HPBREGION_STATE region_state;
+ /* Region number */
+ int region;
+ /* How many subregions in this region */
+ int subregion_count;
+ /* How many dirty subregions in this region */
+ int subregion_dirty_count;
+
+ /* Hit count by the read on this region */
+ unsigned int hit_count;
+ /* List head for active reion list*/
+ struct list_head list_region;
+};
+
+struct ufshpb_map_req {
+ struct ufshpb_lu *hpb;
+ struct ufshpb_map_ctx *mctx;
+ struct request *req;
+ struct bio bio;
+ struct bio_vec bvec[MAX_BVEC_SIZE];/* FIXME */
+ int region;
+ int subregion;
+ int lun;
+ int retry_cnt;
+
+ struct list_head list_map_req;
+
+ /* Follow parameters used by debug/profile */
+ __u64 req_start_t;
+ __u64 workq_enter_t;
+ __u64 req_issue_t;
+ __u64 req_endio_t;
+ __u64 req_end_t;
+};
+
+enum victim_selection_type {
+ /* Select the first active region in list to evict */
+ LRU = 1,
+ /* Choose the active region with lowest hit count to evict */
+ LFU = 2,
+};
+
+/*
+ * local active region info list
+ */
+struct active_region_info {
+ enum victim_selection_type selection_type;
+ /* Active region list */
+ struct list_head lru;
+ /* Maximum active regions supported, doesn't include pinned regions */
+ int max_active_cnt;
+ /* Current Active region count */
+ atomic64_t active_cnt;
+};
+
+struct ufshpb_running_status {
+ atomic64_t hit;
+ atomic64_t miss;
+ atomic64_t region_miss;
+ atomic64_t subregion_miss;
+ atomic64_t entry_dirty_miss;
+ atomic64_t rb_noti_cnt;
+ atomic64_t map_req_cnt;
+ atomic64_t region_add;
+ atomic64_t region_evict;
+ atomic64_t rb_fail;
+};
+
+struct ufshpb_lu {
+ /* The LU number associated with this HPB */
+ int lun;
+ struct ufs_hba *hba;
+ bool lu_hpb_enable;
+ spinlock_t hpb_lock;
+
+ /* HPB region info memory */
+ struct ufshpb_region *region_tbl;
+
+ /* HPB response info, the count = queue_depth */
+ struct ufshpb_rsp_info *rsp_info;
+ struct list_head lh_rsp_info_free;
+ struct list_head lh_rsp_info;
+ spinlock_t rsp_list_lock;
+
+ /* HPB request info, the count = queue_depth */
+ struct ufshpb_req_info *req_info;
+ struct list_head lh_req_info_free;
+ struct list_head lh_req_info;
+ spinlock_t req_list_lock;
+
+ /* HPB L2P mapping request memory, the count = subregions_per_lu */
+ struct ufshpb_map_req *map_req;
+ struct list_head lh_map_req_free;
+
+ /* The map_req list for map_req needed to retry */
+ struct list_head lh_map_req_retry;
+
+ /* mapping memory context = num_regions * subregions_per_region */
+ struct list_head lh_map_ctx;
+ int free_mctx;
+
+ struct list_head lh_subregion_req;
+
+ /* Active regions info list */
+ struct active_region_info lru_info;
+
+ struct work_struct ufshpb_sync_work;
+ struct work_struct ufshpb_req_work;
+ struct delayed_work retry_work;
+ struct work_struct ufshpb_rsp_work;
+
+ int subregions_per_lu;
+ int regions_per_lu;
+ /* entry size in byte per subregion */
+ int subregion_entry_size;
+ int last_subregion_entry_size;
+ int lu_max_active_regions;
+ int entries_per_subregion;
+ int entries_per_subregion_shift;
+ int entries_per_subregion_mask;
+ int entries_per_region_shift;
+ int entries_per_region_mask;
+ int subregions_per_region;
+ int subregions_in_last_region;
+ int dwords_per_subregion;
+ unsigned long long subregion_size;
+ int mpage_bytes;
+ int mpages_per_subregion;
+ /* Total number of addressable logical blocks in the logical unit */
+ int lu_num_blocks;
+
+ /* UFS HPB sysfs */
+ struct kobject kobj;
+ struct mutex sysfs_lock;
+ struct ufshpb_sysfs_entry *sysfs_entries;
+
+ /* for debug */
+ bool force_disable;
+ bool force_map_req_disable;
+ int debug;
+ struct ufshpb_running_status status;
+};
+
+struct ufshpb_sysfs_entry {
+ struct attribute attr;
+ ssize_t (*show)(struct ufshpb_lu *hpb, char *buf);
+ ssize_t (*store)(struct ufshpb_lu *hpb, const char *buf, size_t count);
+};
+
+struct ufshcd_lrb;
+
+void ufshpb_retrieve_l2p_entry(struct ufs_hba *hba, struct ufshcd_lrb *lrbp);
+void ufshpb_rsp_handler(struct ufs_hba *hba, struct ufshcd_lrb *lrbp);
+void ufshpb_release(struct ufs_hba *hba, enum UFSHPB_STATE state);
+void ufshpb_init(struct ufs_hba *hba);
+
+#endif /* End of Header */