// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2013-2021, Linux Foundation. All rights reserved. * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ufshcd.h" #include "ufshcd-pltfrm.h" #include "unipro.h" #include "ufs-qcom.h" #define CREATE_TRACE_POINTS #include "ufs-qcom-trace.h" #include "ufshci.h" #include "ufs_quirks.h" #include "ufshcd-crypto-qti.h" #if IS_ENABLED(CONFIG_QTI_CRYPTO_FDE) #include #endif #define UFS_QCOM_DEFAULT_DBG_PRINT_EN \ (UFS_QCOM_DBG_PRINT_REGS_EN | UFS_QCOM_DBG_PRINT_TEST_BUS_EN) #define UFS_DDR "ufs-ddr" #define CPU_UFS "cpu-ufs" #define MAX_PROP_SIZE 32 #define VDDP_REF_CLK_MIN_UV 1200000 #define VDDP_REF_CLK_MAX_UV 1200000 #define VCCQ_DEFAULT_1_2V 1200000 #define UFS_QCOM_LOAD_MON_DLY_MS 30 #define ANDROID_BOOT_DEV_MAX 30 #define UFS_BOOT_DEVICE 0x4 #define UFS_QCOM_IRQ_PRIME_MASK 0x80 #define UFS_QCOM_IRQ_SLVR_MASK 0x0f #define UFS_QCOM_BER_TH_DEF_G1_G4 0 #define UFS_QCOM_BER_TH_DEF_G5 3 /* * Default time window of PHY BER monitor in millisecond. * Can be overridden by MODULE CmdLine and MODULE sysfs node. */ #define UFS_QCOM_BER_DUR_DEF_MS (60*60*1000) /* Max number of log pages */ #define UFS_QCOM_MAX_LOG_SZ 10 #define ufs_qcom_log_str(host, fmt, ...) \ do { \ if (host->ufs_ipc_log_ctx && host->dbg_en) \ ipc_log_string(host->ufs_ipc_log_ctx, \ ",%d,"fmt, current->cpu, ##__VA_ARGS__);\ } while (0) #define ufs_qcom_msg(level, dev, fmt, ...) \ do { \ if (!IS_ERR_OR_NULL(dev)) { \ switch (level) { \ case ERR: \ dev_err(dev, "ERROR: "fmt, ##__VA_ARGS__); break; \ case WARN: \ dev_warn(dev, "WARN: "fmt, ##__VA_ARGS__); break; \ case DBG: \ dev_dbg(dev, "DEBUG: "fmt, ##__VA_ARGS__); break; \ default: \ dev_info(dev, "INFO: "fmt, ##__VA_ARGS__); \ } \ } \ else { \ switch (level) { \ case ERR: \ pr_err("ERROR: "fmt, ##__VA_ARGS__); break; \ case WARN: \ pr_warn("WARN: "fmt, ##__VA_ARGS__); break; \ case DBG: \ pr_debug("DEBUG: "fmt, ##__VA_ARGS__); break; \ default: \ pr_info("INFO: "fmt, ##__VA_ARGS__); \ } \ } \ } while (0) \ enum { UFS_QCOM_SYSFS_NONE, UFS_QCOM_SYSFS_S2R, UFS_QCOM_SYSFS_DEEPSLEEP, }; static char android_boot_dev[ANDROID_BOOT_DEV_MAX]; static DEFINE_PER_CPU(struct freq_qos_request, qos_min_req); enum { TSTBUS_UAWM, TSTBUS_UARM, TSTBUS_TXUC, TSTBUS_RXUC, TSTBUS_DFC, TSTBUS_TRLUT, TSTBUS_TMRLUT, TSTBUS_OCSC, TSTBUS_UTP_HCI, TSTBUS_COMBINED, TSTBUS_WRAPPER, TSTBUS_UNIPRO, TSTBUS_MAX, }; enum ufs_msg_level { ERR = 3, WARN = 4, INFO = 6, DBG = 7, }; enum { UFS_QCOM_CMD_SEND, UFS_QCOM_CMD_COMPL, }; struct ufs_qcom_dev_params { u32 pwm_rx_gear; /* pwm rx gear to work in */ u32 pwm_tx_gear; /* pwm tx gear to work in */ u32 hs_rx_gear; /* hs rx gear to work in */ u32 hs_tx_gear; /* hs tx gear to work in */ u32 rx_lanes; /* number of rx lanes */ u32 tx_lanes; /* number of tx lanes */ u32 rx_pwr_pwm; /* rx pwm working pwr */ u32 tx_pwr_pwm; /* tx pwm working pwr */ u32 rx_pwr_hs; /* rx hs working pwr */ u32 tx_pwr_hs; /* tx hs working pwr */ u32 hs_rate; /* rate A/B to work in HS */ u32 desired_working_mode; }; static struct ufs_qcom_host *ufs_qcom_hosts[MAX_UFS_QCOM_HOSTS]; static void ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host *host); static int ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(struct ufs_hba *hba, u32 clk_1us_cycles, u32 clk_40ns_cycles); static void ufs_qcom_parse_limits(struct ufs_qcom_host *host); static void ufs_qcom_parse_lpm(struct ufs_qcom_host *host); static void ufs_qcom_parse_wb(struct ufs_qcom_host *host); static int ufs_qcom_set_dme_vs_core_clk_ctrl_max_freq_mode(struct ufs_hba *hba); static int ufs_qcom_init_sysfs(struct ufs_hba *hba); static int ufs_qcom_update_qos_constraints(struct qos_cpu_group *qcg, enum constraint type); static int ufs_qcom_unvote_qos_all(struct ufs_hba *hba); static void ufs_qcom_parse_g4_workaround_flag(struct ufs_qcom_host *host); static int ufs_qcom_mod_min_cpufreq(unsigned int cpu, s32 new_val); static int ufs_qcom_config_shared_ice(struct ufs_qcom_host *host); static int ufs_qcom_ber_threshold_set(const char *val, const struct kernel_param *kp); static int ufs_qcom_ber_duration_set(const char *val, const struct kernel_param *kp); static void ufs_qcom_ber_mon_init(struct ufs_hba *hba); static void ufs_qcom_populate_available_cpus(struct ufs_hba *hba); static s64 idle_time[UFS_QCOM_BER_MODE_MAX]; static ktime_t idle_start; static bool crash_on_ber; static bool override_ber_threshold; static bool override_ber_duration; static unsigned int ber_threshold = UFS_QCOM_BER_TH_DEF_G1_G4; static unsigned int ber_mon_dur_ms = UFS_QCOM_BER_DUR_DEF_MS; static struct ufs_qcom_ber_table ber_table[] = { [UFS_HS_DONT_CHANGE] = {UFS_QCOM_BER_MODE_G1_G4, UFS_QCOM_BER_TH_DEF_G1_G4}, [UFS_HS_G1] = {UFS_QCOM_BER_MODE_G1_G4, UFS_QCOM_BER_TH_DEF_G1_G4}, [UFS_HS_G2] = {UFS_QCOM_BER_MODE_G1_G4, UFS_QCOM_BER_TH_DEF_G1_G4}, [UFS_HS_G3] = {UFS_QCOM_BER_MODE_G1_G4, UFS_QCOM_BER_TH_DEF_G1_G4}, [UFS_HS_G4] = {UFS_QCOM_BER_MODE_G1_G4, UFS_QCOM_BER_TH_DEF_G1_G4}, [UFS_HS_G5] = {UFS_QCOM_BER_MODE_G5, UFS_QCOM_BER_TH_DEF_G5}, }; module_param(crash_on_ber, bool, 0644); MODULE_PARM_DESC(crash_on_ber, "Crash if PHY BER exceeds threshold. the default value is false"); static const struct kernel_param_ops ber_threshold_ops = { .set = ufs_qcom_ber_threshold_set, .get = param_get_uint, }; module_param_cb(ber_threshold, &ber_threshold_ops, &ber_threshold, 0644); MODULE_PARM_DESC(ber_threshold, "Set UFS BER threshold, should be less than 16.\n" "The default value is 3 for G5 and 0 for Non G5."); static int ufs_qcom_ber_threshold_set(const char *val, const struct kernel_param *kp) { unsigned int n; int ret; ret = kstrtou32(val, 0, &n); if (ret != 0 || n > (UFS_QCOM_EVT_LEN - 1)) return -EINVAL; if (n) override_ber_threshold = true; else override_ber_threshold = false; return param_set_uint(val, kp); } static const struct kernel_param_ops ber_duration_ops = { .set = ufs_qcom_ber_duration_set, .get = param_get_uint, }; module_param_cb(ber_duration_ms, &ber_duration_ops, &ber_mon_dur_ms, 0644); MODULE_PARM_DESC(ber_duration_ms, "Set the duration of BER monitor window.\n" "The default value is 3600000(1 hour)"); static int ufs_qcom_ber_duration_set(const char *val, const struct kernel_param *kp) { s64 n; if (kstrtos64(val, 0, &n)) return -EINVAL; if (n) override_ber_duration = true; else override_ber_duration = false; return param_set_uint(val, kp); } /** * ufs_qcom_save_regs - read register value and save to memory for specified domain. * If it is a new domain which never been saved, allocate memory for it and add to list. * @host - ufs_qcom_host * @offset - register address offest * @len - length or size * @prefix - domain name */ static int ufs_qcom_save_regs(struct ufs_qcom_host *host, size_t offset, size_t len, const char *prefix) { struct ufs_qcom_regs *regs = NULL; struct list_head *head = &host->regs_list_head; unsigned int noio_flag; size_t pos; if (offset % 4 != 0 || len % 4 != 0) return -EINVAL; /* find the node if this register domain has been saved before */ list_for_each_entry(regs, head, list) if (regs->prefix && !strcmp(regs->prefix, prefix)) break; /* create a new node and add it to list if this domain never been written */ if (®s->list == head) { /* use memalloc_noio_save() here as GFP_ATOMIC should not be invoked * in an IO error context */ noio_flag = memalloc_noio_save(); regs = devm_kzalloc(host->hba->dev, sizeof(*regs), GFP_ATOMIC); if (!regs) goto out; regs->ptr = devm_kzalloc(host->hba->dev, len, GFP_ATOMIC); if (!regs->ptr) goto out; memalloc_noio_restore(noio_flag); regs->prefix = prefix; regs->len = len; list_add_tail(®s->list, &host->regs_list_head); } for (pos = 0; pos < len; pos += 4) { if (offset == 0 && pos >= REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER && pos <= REG_UIC_ERROR_CODE_DME) continue; regs->ptr[pos / 4] = ufshcd_readl(host->hba, offset + pos); } return 0; out: memalloc_noio_restore(noio_flag); return -ENOMEM; } static int ufs_qcom_save_testbus(struct ufs_qcom_host *host) { struct ufs_qcom_regs *regs = NULL; struct list_head *head = &host->regs_list_head; int i, j; int nminor = 32, testbus_len = nminor * sizeof(u32); unsigned int noio_flag; char *prefix; for (j = 0; j < TSTBUS_MAX; j++) { switch (j) { case TSTBUS_UAWM: prefix = "TSTBUS_UAWM "; break; case TSTBUS_UARM: prefix = "TSTBUS_UARM "; break; case TSTBUS_TXUC: prefix = "TSTBUS_TXUC "; break; case TSTBUS_RXUC: prefix = "TSTBUS_RXUC "; break; case TSTBUS_DFC: prefix = "TSTBUS_DFC "; break; case TSTBUS_TRLUT: prefix = "TSTBUS_TRLUT "; break; case TSTBUS_TMRLUT: prefix = "TSTBUS_TMRLUT "; break; case TSTBUS_OCSC: prefix = "TSTBUS_OCSC "; break; case TSTBUS_UTP_HCI: prefix = "TSTBUS_UTP_HCI "; break; case TSTBUS_COMBINED: prefix = "TSTBUS_COMBINED "; break; case TSTBUS_WRAPPER: prefix = "TSTBUS_WRAPPER "; break; case TSTBUS_UNIPRO: prefix = "TSTBUS_UNIPRO "; break; default: prefix = "UNKNOWN "; break; } /* find the node if this register domain has been saved before */ list_for_each_entry(regs, head, list) if (regs->prefix && !strcmp(regs->prefix, prefix)) break; /* create a new node and add it to list if this domain never been written */ if (®s->list == head) { noio_flag = memalloc_noio_save(); regs = devm_kzalloc(host->hba->dev, sizeof(*regs), GFP_ATOMIC); if (!regs) goto out; regs->ptr = devm_kzalloc(host->hba->dev, testbus_len, GFP_ATOMIC); if (!regs->ptr) goto out; memalloc_noio_restore(noio_flag); regs->prefix = prefix; regs->len = testbus_len; list_add_tail(®s->list, &host->regs_list_head); } host->testbus.select_major = j; for (i = 0; i < nminor; i++) { host->testbus.select_minor = i; ufs_qcom_testbus_config(host); regs->ptr[i] = ufshcd_readl(host->hba, UFS_TEST_BUS); } } return 0; out: memalloc_noio_restore(noio_flag); return -ENOMEM; } static inline void cancel_dwork_unvote_cpufreq(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int err; if (host->cpufreq_dis) return; cancel_delayed_work_sync(&host->fwork); if (!host->cur_freq_vote) return; atomic_set(&host->num_reqs_threshold, 0); err = ufs_qcom_mod_min_cpufreq(host->config_cpu, host->min_cpu_scale_freq); if (err < 0) ufs_qcom_msg(ERR, hba->dev, "fail set cpufreq-fmin_def %d:\n", err); else host->cur_freq_vote = false; ufs_qcom_msg(DBG, hba->dev, "%s,err=%d\n", __func__, err); } static int ufs_qcom_get_pwr_dev_param(struct ufs_qcom_dev_params *qcom_param, struct ufs_pa_layer_attr *dev_max, struct ufs_pa_layer_attr *agreed_pwr) { int min_qcom_gear; int min_dev_gear; bool is_dev_sup_hs = false; bool is_qcom_max_hs = false; if (dev_max->pwr_rx == FAST_MODE) is_dev_sup_hs = true; if (qcom_param->desired_working_mode == FAST) { is_qcom_max_hs = true; min_qcom_gear = min_t(u32, qcom_param->hs_rx_gear, qcom_param->hs_tx_gear); } else { min_qcom_gear = min_t(u32, qcom_param->pwm_rx_gear, qcom_param->pwm_tx_gear); } /* * device doesn't support HS but qcom_param->desired_working_mode is * HS, thus device and qcom_param don't agree */ if (!is_dev_sup_hs && is_qcom_max_hs) { ufs_qcom_msg(ERR, NULL, "%s: failed to agree on power mode (device doesn't support HS but requested power is HS)\n", __func__); return -EOPNOTSUPP; } else if (is_dev_sup_hs && is_qcom_max_hs) { /* * since device supports HS, it supports FAST_MODE. * since qcom_param->desired_working_mode is also HS * then final decision (FAST/FASTAUTO) is done according * to qcom_params as it is the restricting factor */ agreed_pwr->pwr_rx = agreed_pwr->pwr_tx = qcom_param->rx_pwr_hs; } else { /* * here qcom_param->desired_working_mode is PWM. * it doesn't matter whether device supports HS or PWM, * in both cases qcom_param->desired_working_mode will * determine the mode */ agreed_pwr->pwr_rx = agreed_pwr->pwr_tx = qcom_param->rx_pwr_pwm; } /* * we would like tx to work in the minimum number of lanes * between device capability and vendor preferences. * the same decision will be made for rx */ agreed_pwr->lane_tx = min_t(u32, dev_max->lane_tx, qcom_param->tx_lanes); agreed_pwr->lane_rx = min_t(u32, dev_max->lane_rx, qcom_param->rx_lanes); /* device maximum gear is the minimum between device rx and tx gears */ min_dev_gear = min_t(u32, dev_max->gear_rx, dev_max->gear_tx); /* * if both device capabilities and vendor pre-defined preferences are * both HS or both PWM then set the minimum gear to be the chosen * working gear. * if one is PWM and one is HS then the one that is PWM get to decide * what is the gear, as it is the one that also decided previously what * pwr the device will be configured to. */ if ((is_dev_sup_hs && is_qcom_max_hs) || (!is_dev_sup_hs && !is_qcom_max_hs)) agreed_pwr->gear_rx = agreed_pwr->gear_tx = min_t(u32, min_dev_gear, min_qcom_gear); else if (!is_dev_sup_hs) agreed_pwr->gear_rx = agreed_pwr->gear_tx = min_dev_gear; else agreed_pwr->gear_rx = agreed_pwr->gear_tx = min_qcom_gear; agreed_pwr->hs_rate = qcom_param->hs_rate; return 0; } #if defined(CONFIG_UFSFEATURE) static void ufs_vh_prep_fn(void *data, struct ufs_hba *hba, struct request *rq, struct ufshcd_lrb *lrbp, int *err) { ufsf_change_lun(ufs_qcom_get_ufsf(hba), lrbp); *err = ufsf_prep_fn(ufs_qcom_get_ufsf(hba), lrbp); } static void ufs_vh_compl_command(void *data, struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { struct utp_upiu_header *header = &lrbp->ucd_rsp_ptr->header; struct ufsf_feature *ufsf = ufs_qcom_get_ufsf(hba); struct scsi_cmnd *cmd = lrbp->cmd; int scsi_status, result, ocs; if (!cmd) return; ocs = le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS; if (ocs != OCS_SUCCESS) goto check_last_req; result = be32_to_cpu(header->dword_0) >> 24; if (result != UPIU_TRANSACTION_RESPONSE) goto check_last_req; scsi_status = be32_to_cpu(header->dword_1) & MASK_SCSI_STATUS; if (scsi_status != SAM_STAT_GOOD) goto check_last_req; ufsf_upiu_check_for_ccd(lrbp); check_last_req: #if defined(CONFIG_UFSHID) /* Check if it is the last request to be completed */ if (hba->outstanding_tasks || !(hba->outstanding_reqs == (1 << lrbp->task_tag))) return; if (!ufshid_is_hid_req(cmd)) schedule_work(&ufsf->on_idle_work); #endif ; } static void ufs_vh_update_sdev(void *data, struct scsi_device *sdev) { struct ufs_hba *hba = shost_priv(sdev->host); struct ufsf_feature *ufsf = ufs_qcom_get_ufsf(hba); ufsf_slave_configure(ufsf, sdev); } static void ufs_vh_send_command(void *data, struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { struct ufsf_feature *ufsf = ufs_qcom_get_ufsf(hba); ufsf_hid_acc_io_stat(ufsf, lrbp); } #endif static struct ufs_qcom_host *rcdev_to_ufs_host(struct reset_controller_dev *rcd) { return container_of(rcd, struct ufs_qcom_host, rcdev); } static void ufs_qcom_dump_regs_wrapper(struct ufs_hba *hba, int offset, int len, const char *prefix, void *priv) { ufshcd_dump_regs(hba, offset, len * 4, prefix); } static int ufs_qcom_get_connected_tx_lanes(struct ufs_hba *hba, u32 *tx_lanes) { int err = 0; err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), tx_lanes); if (err) ufs_qcom_msg(ERR, hba->dev, "%s: couldn't read PA_CONNECTEDTXDATALANES %d\n", __func__, err); return err; } static int ufs_qcom_get_connected_rx_lanes(struct ufs_hba *hba, u32 *rx_lanes) { int err = 0; err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES), rx_lanes); if (err) ufs_qcom_msg(ERR, hba->dev, "%s: couldn't read PA_CONNECTEDRXDATALANES %d\n", __func__, err); return err; } static int ufs_qcom_host_clk_get(struct device *dev, const char *name, struct clk **clk_out, bool optional) { struct clk *clk; int err = 0; clk = devm_clk_get(dev, name); if (!IS_ERR(clk)) { *clk_out = clk; return 0; } err = PTR_ERR(clk); if (optional && err == -ENOENT) { *clk_out = NULL; return 0; } if (err != -EPROBE_DEFER) ufs_qcom_msg(ERR, dev, "failed to get %s err %d\n", name, err); return err; } static int ufs_qcom_host_clk_enable(struct device *dev, const char *name, struct clk *clk) { int err = 0; err = clk_prepare_enable(clk); if (err) ufs_qcom_msg(ERR, dev, "%s: %s enable failed %d\n", __func__, name, err); return err; } static void ufs_qcom_disable_lane_clks(struct ufs_qcom_host *host) { if (!host->is_lane_clks_enabled) return; if (host->tx_l1_sync_clk) clk_disable_unprepare(host->tx_l1_sync_clk); clk_disable_unprepare(host->tx_l0_sync_clk); if (host->rx_l1_sync_clk) clk_disable_unprepare(host->rx_l1_sync_clk); clk_disable_unprepare(host->rx_l0_sync_clk); host->is_lane_clks_enabled = false; } static int ufs_qcom_enable_lane_clks(struct ufs_qcom_host *host) { int err = 0; struct device *dev = host->hba->dev; if (host->is_lane_clks_enabled) return 0; err = ufs_qcom_host_clk_enable(dev, "rx_lane0_sync_clk", host->rx_l0_sync_clk); if (err) goto out; err = ufs_qcom_host_clk_enable(dev, "tx_lane0_sync_clk", host->tx_l0_sync_clk); if (err) goto disable_rx_l0; if (host->hba->lanes_per_direction > 1) { err = ufs_qcom_host_clk_enable(dev, "rx_lane1_sync_clk", host->rx_l1_sync_clk); if (err) goto disable_tx_l0; /* The tx lane1 clk could be muxed, hence keep this optional */ if (host->tx_l1_sync_clk) { err = ufs_qcom_host_clk_enable(dev, "tx_lane1_sync_clk", host->tx_l1_sync_clk); if (err) goto disable_rx_l1; } } host->is_lane_clks_enabled = true; goto out; disable_rx_l1: clk_disable_unprepare(host->rx_l1_sync_clk); disable_tx_l0: clk_disable_unprepare(host->tx_l0_sync_clk); disable_rx_l0: clk_disable_unprepare(host->rx_l0_sync_clk); out: return err; } static int ufs_qcom_init_lane_clks(struct ufs_qcom_host *host) { int err = 0; struct device *dev = host->hba->dev; if (has_acpi_companion(dev)) return 0; err = ufs_qcom_host_clk_get(dev, "rx_lane0_sync_clk", &host->rx_l0_sync_clk, false); if (err) { ufs_qcom_msg(ERR, dev, "%s: failed to get rx_lane0_sync_clk, err %d\n", __func__, err); goto out; } err = ufs_qcom_host_clk_get(dev, "tx_lane0_sync_clk", &host->tx_l0_sync_clk, false); if (err) { ufs_qcom_msg(ERR, dev, "%s: failed to get tx_lane0_sync_clk, err %d\n", __func__, err); goto out; } /* In case of single lane per direction, don't read lane1 clocks */ if (host->hba->lanes_per_direction > 1) { err = ufs_qcom_host_clk_get(dev, "rx_lane1_sync_clk", &host->rx_l1_sync_clk, false); if (err) { ufs_qcom_msg(ERR, dev, "%s: failed to get rx_lane1_sync_clk, err %d\n", __func__, err); goto out; } err = ufs_qcom_host_clk_get(dev, "tx_lane1_sync_clk", &host->tx_l1_sync_clk, true); } out: return err; } static int ufs_qcom_link_startup_post_change(struct ufs_hba *hba) { u32 tx_lanes; int err = 0; struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct phy *phy = host->generic_phy; err = ufs_qcom_get_connected_tx_lanes(hba, &tx_lanes); if (err) goto out; ufs_qcom_phy_set_tx_lane_enable(phy, tx_lanes); /* * Some UFS devices send incorrect LineCfg data as part of power mode * change sequence which may cause host PHY to go into bad state. * Disabling Rx LineCfg of host PHY should help avoid this. */ if (ufshcd_get_local_unipro_ver(hba) == UFS_UNIPRO_VER_1_41) ufs_qcom_phy_ctrl_rx_linecfg(phy, false); /* * UFS controller has *clk_req output to GCC, for each of the clocks * entering it. When *clk_req for a specific clock is de-asserted, * a corresponding clock from GCC is stopped. UFS controller de-asserts * *clk_req outputs when it is in Auto Hibernate state only if the * Clock request feature is enabled. * Enable the Clock request feature: * - Enable HW clock control for UFS clocks in GCC (handled by the * clock driver as part of clk_prepare_enable). * - Set the AH8_CFG.*CLK_REQ register bits to 1. */ if (ufshcd_is_auto_hibern8_supported(hba)) ufshcd_writel(hba, ufshcd_readl(hba, UFS_AH8_CFG) | UFS_HW_CLK_CTRL_EN, UFS_AH8_CFG); /* * Make sure clock request feature gets enabled for HW clk gating * before further operations. */ mb(); out: return err; } static int ufs_qcom_check_hibern8(struct ufs_hba *hba) { int err; u32 tx_fsm_val = 0; unsigned long timeout = jiffies + msecs_to_jiffies(HBRN8_POLL_TOUT_MS); do { err = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(MPHY_TX_FSM_STATE, UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)), &tx_fsm_val); if (err || tx_fsm_val == TX_FSM_HIBERN8) break; /* sleep for max. 200us */ usleep_range(100, 200); } while (time_before(jiffies, timeout)); /* * we might have scheduled out for long during polling so * check the state again. */ if (time_after(jiffies, timeout)) err = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(MPHY_TX_FSM_STATE, UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)), &tx_fsm_val); if (err) { ufs_qcom_msg(ERR, hba->dev, "%s: unable to get TX_FSM_STATE, err %d\n", __func__, err); } else if (tx_fsm_val != TX_FSM_HIBERN8) { err = tx_fsm_val; ufs_qcom_msg(ERR, hba->dev, "%s: invalid TX_FSM_STATE = %d\n", __func__, err); } return err; } static void ufs_qcom_select_unipro_mode(struct ufs_qcom_host *host) { ufshcd_rmwl(host->hba, QUNIPRO_SEL, ufs_qcom_cap_qunipro(host) ? QUNIPRO_SEL : 0, REG_UFS_CFG1); if (host->hw_ver.major >= 0x05) ufshcd_rmwl(host->hba, QUNIPRO_G4_SEL, 0, REG_UFS_CFG0); } /* * ufs_qcom_host_reset - reset host controller and PHY */ static int ufs_qcom_host_reset(struct ufs_hba *hba) { int ret = 0; struct ufs_qcom_host *host = ufshcd_get_variant(hba); bool reenable_intr = false; host->reset_in_progress = true; if (!host->core_reset) { ufs_qcom_msg(WARN, hba->dev, "%s: reset control not set\n", __func__); goto out; } reenable_intr = hba->is_irq_enabled; disable_irq(hba->irq); hba->is_irq_enabled = false; ret = reset_control_assert(host->core_reset); if (ret) { ufs_qcom_msg(ERR, hba->dev, "%s: core_reset assert failed, err = %d\n", __func__, ret); goto out; } /* * The hardware requirement for delay between assert/deassert * is at least 3-4 sleep clock (32.7KHz) cycles, which comes to * ~125us (4/32768). To be on the safe side add 200us delay. */ usleep_range(200, 210); ret = reset_control_deassert(host->core_reset); if (ret) ufs_qcom_msg(ERR, hba->dev, "%s: core_reset deassert failed, err = %d\n", __func__, ret); usleep_range(1000, 1100); /* * The ice registers are also reset to default values after a ufs * host controller reset. Reset the ice internal software flags here * so that the ice hardware will be re-initialized properly in the * later part of the UFS host controller reset. */ ufs_qcom_ice_disable(host); if (reenable_intr) { enable_irq(hba->irq); hba->is_irq_enabled = true; } out: host->vdd_hba_pc = false; host->reset_in_progress = false; return ret; } static int ufs_qcom_phy_power_on(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct phy *phy = host->generic_phy; int ret = 0; mutex_lock(&host->phy_mutex); if (!host->is_phy_pwr_on) { ret = phy_power_on(phy); if (ret) { mutex_unlock(&host->phy_mutex); return ret; } host->is_phy_pwr_on = true; } mutex_unlock(&host->phy_mutex); return ret; } static int ufs_qcom_phy_power_off(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct phy *phy = host->generic_phy; int ret = 0; mutex_lock(&host->phy_mutex); if (host->is_phy_pwr_on) { ret = phy_power_off(phy); if (ret) { mutex_unlock(&host->phy_mutex); return ret; } host->is_phy_pwr_on = false; } mutex_unlock(&host->phy_mutex); return ret; } static int ufs_qcom_power_up_sequence(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct phy *phy = host->generic_phy; int ret = 0; enum phy_mode mode = (host->limit_rate == PA_HS_MODE_B) ? PHY_MODE_UFS_HS_B : PHY_MODE_UFS_HS_A; int submode = host->limit_phy_submode; if (host->hw_ver.major < 0x4) submode = UFS_QCOM_PHY_SUBMODE_NON_G4; phy_set_mode_ext(phy, mode, submode); ret = ufs_qcom_phy_power_on(hba); if (ret) { ufs_qcom_msg(ERR, hba->dev, "%s: phy power on failed, ret = %d\n", __func__, ret); goto out; } ret = phy_calibrate(phy); if (ret) { ufs_qcom_msg(ERR, hba->dev, "%s: Failed to calibrate PHY %d\n", __func__, ret); goto out; } ufs_qcom_select_unipro_mode(host); out: return ret; } /* * The UTP controller has a number of internal clock gating cells (CGCs). * Internal hardware sub-modules within the UTP controller control the CGCs. * Hardware CGCs disable the clock to inactivate UTP sub-modules not involved * in a specific operation, UTP controller CGCs are by default disabled and * this function enables them (after every UFS link startup) to save some power * leakage. * * UFS host controller v3.0.0 onwards has internal clock gating mechanism * in Qunipro, enable them to save additional power. */ static int ufs_qcom_enable_hw_clk_gating(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int err = 0; /* Enable UTP internal clock gating */ ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_CFG2) | REG_UFS_CFG2_CGC_EN_ALL, REG_UFS_CFG2); if (host->hw_ver.major == 0x05) /* Ensure unused Unipro block's clock is gated */ ufshcd_rmwl(host->hba, UNUSED_UNIPRO_CLK_GATED, UNUSED_UNIPRO_CLK_GATED, UFS_AH8_CFG); /* Ensure that HW clock gating is enabled before next operations */ mb(); /* Enable Qunipro internal clock gating if supported */ if (!ufs_qcom_cap_qunipro_clk_gating(host)) goto out; /* Enable all the mask bits */ err = ufshcd_dme_rmw(hba, DL_VS_CLK_CFG_MASK, DL_VS_CLK_CFG_MASK, DL_VS_CLK_CFG); if (err) goto out; err = ufshcd_dme_rmw(hba, PA_VS_CLK_CFG_REG_MASK, PA_VS_CLK_CFG_REG_MASK, PA_VS_CLK_CFG_REG); if (err) goto out; if (!((host->hw_ver.major == 4) && (host->hw_ver.minor == 0) && (host->hw_ver.step == 0))) { err = ufshcd_dme_rmw(hba, DME_VS_CORE_CLK_CTRL_DME_HW_CGC_EN, DME_VS_CORE_CLK_CTRL_DME_HW_CGC_EN, DME_VS_CORE_CLK_CTRL); } else { ufs_qcom_msg(ERR, hba->dev, "%s: skipping DME_HW_CGC_EN set\n", __func__); } out: return err; } static void ufs_qcom_force_mem_config(struct ufs_hba *hba) { struct ufs_clk_info *clki; /* * Configure the behavior of ufs clocks core and peripheral * memory state when they are turned off. * This configuration is required to allow retaining * ICE crypto configuration (including keys) when * core_clk_ice is turned off, and powering down * non-ICE RAMs of host controller. * * This is applicable only to gcc clocks. */ list_for_each_entry(clki, &hba->clk_list_head, list) { /* skip it for non-gcc (rpmh) clocks */ if (!strcmp(clki->name, "ref_clk")) continue; if (!strcmp(clki->name, "core_clk_ice") || !strcmp(clki->name, "core_clk_ice_hw_ctl")) qcom_clk_set_flags(clki->clk, CLKFLAG_RETAIN_MEM); else qcom_clk_set_flags(clki->clk, CLKFLAG_NORETAIN_MEM); qcom_clk_set_flags(clki->clk, CLKFLAG_NORETAIN_PERIPH); qcom_clk_set_flags(clki->clk, CLKFLAG_PERIPH_OFF_CLEAR); } ufshcd_readl(hba, REG_UFS_CFG2); } static int ufs_qcom_hce_enable_notify(struct ufs_hba *hba, enum ufs_notify_change_status status) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int err = 0; switch (status) { case PRE_CHANGE: ufs_qcom_force_mem_config(hba); ufs_qcom_power_up_sequence(hba); /* * The PHY PLL output is the source of tx/rx lane symbol * clocks, hence, enable the lane clocks only after PHY * is initialized. */ err = ufs_qcom_enable_lane_clks(host); if (err) ufs_qcom_msg(ERR, hba->dev, "%s: enable lane clks failed, ret=%d\n", __func__, err); /* * ICE enable needs to be called before ufshcd_crypto_enable * during resume as it is needed before reprogramming all * keys. So moving it to PRE_CHANGE. */ ufs_qcom_ice_enable(host); break; case POST_CHANGE: err = ufs_qcom_config_shared_ice(host); if (err) { ufs_qcom_msg(ERR, hba->dev, "%s: config shared ice failed, ret=%d\n", __func__, err); break; } /* check if UFS PHY moved from DISABLED to HIBERN8 */ err = ufs_qcom_check_hibern8(hba); ufs_qcom_enable_hw_clk_gating(hba); break; default: ufs_qcom_msg(ERR, hba->dev, "%s: invalid status %d\n", __func__, status); err = -EINVAL; break; } ufs_qcom_log_str(host, "-,%d,%d\n", status, err); if (host->dbg_en) trace_ufs_qcom_hce_enable_notify(dev_name(hba->dev), status, err); return err; } /* * Returns zero for success and non-zero in case of a failure */ static int __ufs_qcom_cfg_timers(struct ufs_hba *hba, u32 gear, u32 hs, u32 rate, bool update_link_startup_timer, bool is_pre_scale_up) { int ret = 0; struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct ufs_clk_info *clki; u32 core_clk_period_in_ns; u32 tx_clk_cycles_per_us = 0; unsigned long core_clk_rate = 0; u32 core_clk_cycles_per_us = 0; static u32 pwm_fr_table[][2] = { {UFS_PWM_G1, 0x1}, {UFS_PWM_G2, 0x1}, {UFS_PWM_G3, 0x1}, {UFS_PWM_G4, 0x1}, }; static u32 hs_fr_table_rA[][2] = { {UFS_HS_G1, 0x1F}, {UFS_HS_G2, 0x3e}, {UFS_HS_G3, 0x7D}, }; static u32 hs_fr_table_rB[][2] = { {UFS_HS_G1, 0x24}, {UFS_HS_G2, 0x49}, {UFS_HS_G3, 0x92}, }; /* * The Qunipro controller does not use following registers: * SYS1CLK_1US_REG, TX_SYMBOL_CLK_1US_REG, CLK_NS_REG & * UFS_REG_PA_LINK_STARTUP_TIMER * But UTP controller uses SYS1CLK_1US_REG register for Interrupt * Aggregation logic / Auto hibern8 logic. * It is mandatory to write SYS1CLK_1US_REG register on UFS host * controller V4.0.0 onwards. */ if (ufs_qcom_cap_qunipro(host) && (!(ufshcd_is_intr_aggr_allowed(hba) || ufshcd_is_auto_hibern8_supported(hba) || host->hw_ver.major >= 4))) goto out; if (gear == 0) { ufs_qcom_msg(ERR, hba->dev, "%s: invalid gear = %d\n", __func__, gear); goto out_error; } list_for_each_entry(clki, &hba->clk_list_head, list) { if (!strcmp(clki->name, "core_clk")) { if (is_pre_scale_up) core_clk_rate = clki->max_freq; else core_clk_rate = clk_get_rate(clki->clk); } } /* If frequency is smaller than 1MHz, set to 1MHz */ if (core_clk_rate < DEFAULT_CLK_RATE_HZ) core_clk_rate = DEFAULT_CLK_RATE_HZ; core_clk_cycles_per_us = core_clk_rate / USEC_PER_SEC; if (ufshcd_readl(hba, REG_UFS_SYS1CLK_1US) != core_clk_cycles_per_us) { ufshcd_writel(hba, core_clk_cycles_per_us, REG_UFS_SYS1CLK_1US); /* * make sure above write gets applied before we return from * this function. */ ufshcd_readl(hba, REG_UFS_SYS1CLK_1US); } if (ufs_qcom_cap_qunipro(host)) goto out; core_clk_period_in_ns = NSEC_PER_SEC / core_clk_rate; core_clk_period_in_ns <<= OFFSET_CLK_NS_REG; core_clk_period_in_ns &= MASK_CLK_NS_REG; switch (hs) { case FASTAUTO_MODE: case FAST_MODE: if (rate == PA_HS_MODE_A) { if (gear > ARRAY_SIZE(hs_fr_table_rA)) { ufs_qcom_msg(ERR, hba->dev, "%s: index %d exceeds table size %zu\n", __func__, gear, ARRAY_SIZE(hs_fr_table_rA)); goto out_error; } tx_clk_cycles_per_us = hs_fr_table_rA[gear-1][1]; } else if (rate == PA_HS_MODE_B) { if (gear > ARRAY_SIZE(hs_fr_table_rB)) { ufs_qcom_msg(ERR, hba->dev, "%s: index %d exceeds table size %zu\n", __func__, gear, ARRAY_SIZE(hs_fr_table_rB)); goto out_error; } tx_clk_cycles_per_us = hs_fr_table_rB[gear-1][1]; } else { ufs_qcom_msg(ERR, hba->dev, "%s: invalid rate = %d\n", __func__, rate); goto out_error; } break; case SLOWAUTO_MODE: case SLOW_MODE: if (gear > ARRAY_SIZE(pwm_fr_table)) { ufs_qcom_msg(ERR, hba->dev, "%s: index %d exceeds table size %zu\n", __func__, gear, ARRAY_SIZE(pwm_fr_table)); goto out_error; } tx_clk_cycles_per_us = pwm_fr_table[gear-1][1]; break; case UNCHANGED: default: ufs_qcom_msg(ERR, hba->dev, "%s: invalid mode = %d\n", __func__, hs); goto out_error; } if (ufshcd_readl(hba, REG_UFS_TX_SYMBOL_CLK_NS_US) != (core_clk_period_in_ns | tx_clk_cycles_per_us)) { /* this register 2 fields shall be written at once */ ufshcd_writel(hba, core_clk_period_in_ns | tx_clk_cycles_per_us, REG_UFS_TX_SYMBOL_CLK_NS_US); /* * make sure above write gets applied before we return from * this function. */ mb(); } if (update_link_startup_timer && host->hw_ver.major != 0x5) { ufshcd_writel(hba, ((core_clk_rate / MSEC_PER_SEC) * 100), REG_UFS_CFG0); /* * make sure that this configuration is applied before * we return */ mb(); } goto out; out_error: ret = -EINVAL; out: return ret; } static int ufs_qcom_cfg_timers(struct ufs_hba *hba, u32 gear, u32 hs, u32 rate, bool update_link_startup_timer) { return __ufs_qcom_cfg_timers(hba, gear, hs, rate, update_link_startup_timer, false); } static int ufs_qcom_set_dme_vs_core_clk_ctrl_max_freq_mode(struct ufs_hba *hba) { struct ufs_clk_info *clki; struct list_head *head = &hba->clk_list_head; u32 max_freq = 0; int err = 0; list_for_each_entry(clki, head, list) { if (!IS_ERR_OR_NULL(clki->clk) && (!strcmp(clki->name, "core_clk_unipro"))) { max_freq = clki->max_freq; break; } } switch (max_freq) { case 300000000: err = ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(hba, 300, 12); break; case 150000000: err = ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(hba, 150, 6); break; default: err = -EINVAL; break; } return err; } /** * ufs_qcom_bypass_cfgready_signal - Tunes PA_VS_CONFIG_REG1 and * PA_VS_CONFIG_REG2 vendor specific attributes of local unipro * to bypass CFGREADY signal on Config interface between UFS * controller and PHY. * * The issue is related to config signals sampling from PHY * to controller. The PHY signals which are driven by 150MHz * clock and sampled by 300MHz instead of 150MHZ. * * The issue will be seen when only one of tx_cfg_rdyn_0 * and tx_cfg_rdyn_1 is 0 around sampling clock edge and * if timing is not met as timing margin for some devices is * very less in one of the corner. * * To workaround this issue, controller should bypass the Cfgready * signal(TX_CFGREADY and RX_CFGREDY) because controller still wait * for another signal tx_savestatusn which will serve same purpose. * * The corresponding HW CR: 'QCTDD06985523' UFS HSG4 test fails * in SDF MAX GLS is linked to this issue. */ static int ufs_qcom_bypass_cfgready_signal(struct ufs_hba *hba) { int err = 0; u32 pa_vs_config_reg1; u32 pa_vs_config_reg2; u32 mask; err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_VS_CONFIG_REG1), &pa_vs_config_reg1); if (err) goto out; err = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_VS_CONFIG_REG1), (pa_vs_config_reg1 | BIT_TX_EOB_COND)); if (err) goto out; err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_VS_CONFIG_REG2), &pa_vs_config_reg2); if (err) goto out; mask = (BIT_RX_EOB_COND | BIT_LINKCFG_WAIT_LL1_RX_CFG_RDY | H8_ENTER_COND_MASK); pa_vs_config_reg2 = (pa_vs_config_reg2 & ~mask) | (0x2 << H8_ENTER_COND_OFFSET); err = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_VS_CONFIG_REG2), (pa_vs_config_reg2)); out: return err; } static void ufs_qcom_dump_attribs(struct ufs_hba *hba) { int ret; int attrs[] = {0x15a0, 0x1552, 0x1553, 0x1554, 0x1555, 0x1556, 0x1557, 0x155a, 0x155b, 0x155c, 0x155d, 0x155e, 0x155f, 0x1560, 0x1561, 0x1568, 0x1569, 0x156a, 0x1571, 0x1580, 0x1581, 0x1583, 0x1584, 0x1585, 0x1586, 0x1587, 0x1590, 0x1591, 0x15a1, 0x15a2, 0x15a3, 0x15a4, 0x15a5, 0x15a6, 0x15a7, 0x15a8, 0x15a9, 0x15aa, 0x15ab, 0x15c0, 0x15c1, 0x15c2, 0x15d0, 0x15d1, 0x15d2, 0x15d3, 0x15d4, 0x15d5, }; int cnt = ARRAY_SIZE(attrs); int i = 0, val; for (; i < cnt; i++) { ret = ufshcd_dme_get(hba, UIC_ARG_MIB(attrs[i]), &val); if (ret) { ufs_qcom_msg(ERR, hba->dev, "Failed reading: 0x%04x, ret:%d\n", attrs[i], ret); continue; } ufs_qcom_msg(ERR, hba->dev, "0x%04x: %d\n", attrs[i], val); } } static void ufs_qcom_validate_link_params(struct ufs_hba *hba) { int val = 0; bool err = false; WARN_ON(ufs_qcom_get_connected_tx_lanes(hba, &val)); if (val != hba->lanes_per_direction) { ufs_qcom_msg(ERR, hba->dev, "%s: Tx lane mismatch [config,reported] [%d,%d]\n", __func__, hba->lanes_per_direction, val); WARN_ON(1); err = true; } val = 0; WARN_ON(ufs_qcom_get_connected_rx_lanes(hba, &val)); if (val != hba->lanes_per_direction) { ufs_qcom_msg(ERR, hba->dev, "%s: Rx lane mismatch [config,reported] [%d,%d]\n", __func__, hba->lanes_per_direction, val); WARN_ON(1); err = true; } if (err) ufs_qcom_dump_attribs(hba); } static int ufs_qcom_link_startup_notify(struct ufs_hba *hba, enum ufs_notify_change_status status) { int err = 0; struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct phy *phy = host->generic_phy; struct device *dev = hba->dev; struct device_node *np = dev->of_node; u32 temp; switch (status) { case PRE_CHANGE: if (!of_property_read_bool(np, "secondary-storage") && strlen(android_boot_dev) && strcmp(android_boot_dev, dev_name(dev))) return -ENODEV; ufshcd_dme_set(hba, UIC_ARG_MIB(PA_AVAILTXDATALANES), hba->lanes_per_direction); ufshcd_dme_set(hba, UIC_ARG_MIB(PA_AVAILRXDATALANES), hba->lanes_per_direction); if (ufs_qcom_cfg_timers(hba, UFS_PWM_G1, SLOWAUTO_MODE, 0, true)) { ufs_qcom_msg(ERR, hba->dev, "%s: ufs_qcom_cfg_timers() failed\n", __func__); err = -EINVAL; goto out; } ufs_qcom_phy_ctrl_rx_linecfg(phy, true); if (ufs_qcom_cap_qunipro(host)) { err = ufs_qcom_set_dme_vs_core_clk_ctrl_max_freq_mode( hba); if (err) goto out; } err = ufs_qcom_enable_hw_clk_gating(hba); if (err) goto out; /* * Controller checks ICE configuration error without * checking if the command is SCSI command */ temp = readl_relaxed(host->dev_ref_clk_ctrl_mmio); temp |= BIT(31); writel_relaxed(temp, host->dev_ref_clk_ctrl_mmio); /* ensure that UTP_SCASI_CHECK_DIS is enabled before link startup */ wmb(); /* * Some UFS devices (and may be host) have issues if LCC is * enabled. So we are setting PA_Local_TX_LCC_Enable to 0 * before link startup which will make sure that both host * and device TX LCC are disabled once link startup is * completed. */ if (ufshcd_get_local_unipro_ver(hba) != UFS_UNIPRO_VER_1_41) err = ufshcd_disable_host_tx_lcc(hba); if (err) goto out; if (host->bypass_g4_cfgready) err = ufs_qcom_bypass_cfgready_signal(hba); break; case POST_CHANGE: ufs_qcom_link_startup_post_change(hba); ufs_qcom_validate_link_params(hba); break; default: break; } out: ufs_qcom_log_str(host, "*,%d,%d\n", status, err); if (host->dbg_en) trace_ufs_qcom_link_startup_notify(dev_name(hba->dev), status, err); return err; } static int ufs_qcom_config_vreg(struct device *dev, struct ufs_vreg *vreg, bool on) { int ret = 0; struct regulator *reg; int min_uV, uA_load; if (!vreg) { WARN_ON(1); ret = -EINVAL; goto out; } reg = vreg->reg; if (regulator_count_voltages(reg) > 0) { uA_load = on ? vreg->max_uA : 0; ret = regulator_set_load(vreg->reg, uA_load); if (ret) goto out; if (vreg->min_uV && vreg->max_uV) { min_uV = on ? vreg->min_uV : 0; ret = regulator_set_voltage(reg, min_uV, vreg->max_uV); if (ret) { ufs_qcom_msg(ERR, dev, "%s: %s failed, err=%d\n", __func__, vreg->name, ret); goto out; } } } out: return ret; } static int ufs_qcom_enable_vreg(struct device *dev, struct ufs_vreg *vreg) { int ret = 0; if (vreg->enabled) return ret; ret = ufs_qcom_config_vreg(dev, vreg, true); if (ret) goto out; ret = regulator_enable(vreg->reg); if (ret) goto out; vreg->enabled = true; out: return ret; } static int ufs_qcom_disable_vreg(struct device *dev, struct ufs_vreg *vreg) { int ret = 0; if (!vreg->enabled) return ret; ret = regulator_disable(vreg->reg); if (ret) goto out; ret = ufs_qcom_config_vreg(dev, vreg, false); if (ret) goto out; vreg->enabled = false; out: return ret; } static int ufs_qcom_mod_min_cpufreq(unsigned int cpu, s32 new_val) { int ret = 0; struct freq_qos_request *qos_req; cpus_read_lock(); if (cpu_online(cpu)) { qos_req = &per_cpu(qos_min_req, cpu); ret = freq_qos_update_request(qos_req, new_val); } cpus_read_unlock(); return ret; } static int ufs_qcom_init_cpu_minfreq_req(struct ufs_qcom_host *host, unsigned int cpu) { int ret; struct cpufreq_policy *policy; struct freq_qos_request *req; policy = cpufreq_cpu_get(cpu); if (!policy) { ufs_qcom_msg(ERR, host->hba->dev, "Failed to get cpu(%u)freq policy\n", cpu); return -EINVAL; } req = &per_cpu(qos_min_req, cpu); ret = freq_qos_add_request(&policy->constraints, req, FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); if (ret < 0) ufs_qcom_msg(ERR, host->hba->dev, "Failed to add freq qos req\n"); cpufreq_cpu_put(policy); return ret; } static void ufs_qcom_set_affinity_hint(struct ufs_hba *hba, bool prime) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); unsigned int set = 0; unsigned int clear = 0; int ret; cpumask_t *affinity_mask; if (prime) { set = IRQ_NO_BALANCING; affinity_mask = &host->perf_mask; } else { clear = IRQ_NO_BALANCING; affinity_mask = &host->silver_mask; } irq_modify_status(hba->irq, clear, set); ret = irq_set_affinity_hint(hba->irq, affinity_mask); if (ret < 0) ufs_qcom_msg(ERR, host->hba->dev, "prime=%d, err=%d\n", prime, ret); } static void ufs_qcom_toggle_pri_affinity(struct ufs_hba *hba, bool on) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); if (on && atomic_read(&host->therm_mitigation)) return; atomic_set(&host->hi_pri_en, on); ufs_qcom_set_affinity_hint(hba, on); } static void ufs_qcom_cpufreq_dwork(struct work_struct *work) { struct ufs_qcom_host *host = container_of(to_delayed_work(work), struct ufs_qcom_host, fwork); unsigned long cur_thres = atomic_read(&host->num_reqs_threshold); unsigned int freq_val = -1; int err = -1; atomic_set(&host->num_reqs_threshold, 0); if (cur_thres > NUM_REQS_HIGH_THRESH && !host->cur_freq_vote) { freq_val = host->max_cpu_scale_freq; if (host->irq_affinity_support) ufs_qcom_toggle_pri_affinity(host->hba, true); } else if (cur_thres < NUM_REQS_LOW_THRESH && host->cur_freq_vote) { freq_val = host->min_cpu_scale_freq; if (host->irq_affinity_support) ufs_qcom_toggle_pri_affinity(host->hba, false); } if (freq_val == -1) goto out; err = ufs_qcom_mod_min_cpufreq(host->config_cpu, freq_val); if (err < 0) ufs_qcom_msg(ERR, host->hba->dev, "fail set cpufreq-fmin to %d: %u\n", err, freq_val); else if (freq_val == host->max_cpu_scale_freq) host->cur_freq_vote = true; else if (freq_val == host->min_cpu_scale_freq) host->cur_freq_vote = false; ufs_qcom_msg(DBG, host->hba->dev, "cur_freq_vote=%d,freq_val=%u,cth=%u\n", host->cur_freq_vote, freq_val, cur_thres); out: queue_delayed_work(host->ufs_qos->workq, &host->fwork, msecs_to_jiffies(UFS_QCOM_LOAD_MON_DLY_MS)); } static int add_group_qos(struct qos_cpu_group *qcg, enum constraint type) { int cpu, err; struct dev_pm_qos_request *qos_req = qcg->qos_req; for_each_cpu(cpu, &qcg->mask) { ufs_qcom_msg(DBG, qcg->host->hba->dev, "%s: cpu: %d | mask: 0x%08x | assoc-qos-req: 0x%08x\n", __func__, cpu, qcg->mask, qos_req); memset(qos_req, 0, sizeof(struct dev_pm_qos_request)); err = dev_pm_qos_add_request(get_cpu_device(cpu), qos_req, DEV_PM_QOS_RESUME_LATENCY, type); if (err < 0) return err; qos_req++; } return 0; } static int remove_group_qos(struct qos_cpu_group *qcg) { int err, cpu; struct dev_pm_qos_request *qos_req = qcg->qos_req; for_each_cpu(cpu, &qcg->mask) { if (!dev_pm_qos_request_active(qos_req)) { qos_req++; continue; } err = dev_pm_qos_remove_request(qos_req); if (err < 0) return err; qos_req++; } return 0; } static void ufs_qcom_device_reset_ctrl(struct ufs_hba *hba, bool asserted) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); /* reset gpio is optional */ if (!host->device_reset) return; gpiod_set_value_cansleep(host->device_reset, asserted); } static int ufs_qcom_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op, enum ufs_notify_change_status status) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int err = 0; if (status == PRE_CHANGE) { #if defined(CONFIG_UFSFEATURE) ufsf_suspend(ufs_qcom_get_ufsf(hba)); #endif return 0; } /* * If UniPro link is not active or OFF, PHY ref_clk, main PHY analog * power rail and low noise analog power rail for PLL can be * switched off. */ if (!ufs_qcom_is_link_active(hba)) { ufs_qcom_disable_lane_clks(host); if (host->vddp_ref_clk && ufs_qcom_is_link_off(hba)) err = ufs_qcom_disable_vreg(hba->dev, host->vddp_ref_clk); if (host->vccq_parent && !hba->auto_bkops_enabled) ufs_qcom_disable_vreg(hba->dev, host->vccq_parent); if (!err) err = ufs_qcom_unvote_qos_all(hba); } if (!err && ufs_qcom_is_link_off(hba) && host->device_reset) ufs_qcom_device_reset_ctrl(hba, true); else if (err) goto out; if (host->vccq_lpm_uV && !ufs_qcom_is_ufs_dev_active(hba) && ufs_qcom_is_link_hibern8(hba)) { err = regulator_set_voltage(hba->vreg_info.vccq->reg, host->vccq_lpm_uV, host->vccq_lpm_uV); if (err) ufs_qcom_msg(ERR, hba->dev, "%s:vccq set %duV failed\n", __func__, host->vccq_lpm_uV); } out: ufs_qcom_log_str(host, "&,%d,%d,%d,%d,%d,%d\n", pm_op, hba->rpm_lvl, hba->spm_lvl, hba->uic_link_state, hba->curr_dev_pwr_mode, err); if (host->dbg_en) trace_ufs_qcom_suspend(dev_name(hba->dev), pm_op, hba->rpm_lvl, hba->spm_lvl, hba->uic_link_state, hba->curr_dev_pwr_mode, err); ufs_qcom_ice_disable(host); cancel_dwork_unvote_cpufreq(hba); return err; } static int ufs_qcom_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int err; #if defined(CONFIG_UFSFEATURE) struct ufsf_feature *ufsf = ufs_qcom_get_ufsf(hba); schedule_work(&ufsf->resume_work); #endif if (host->vddp_ref_clk && (hba->rpm_lvl > UFS_PM_LVL_3 || hba->spm_lvl > UFS_PM_LVL_3)) ufs_qcom_enable_vreg(hba->dev, host->vddp_ref_clk); if (host->vccq_parent) ufs_qcom_enable_vreg(hba->dev, host->vccq_parent); err = ufs_qcom_enable_lane_clks(host); if (err) goto out; if (host->vccq_lpm_uV) { err = regulator_set_voltage(hba->vreg_info.vccq->reg, VCCQ_DEFAULT_1_2V, VCCQ_DEFAULT_1_2V); if (err) ufs_qcom_msg(ERR, hba->dev, "%s:vccq set 1.2V failed\n", __func__); } out: ufs_qcom_log_str(host, "$,%d,%d,%d,%d,%d,%d\n", pm_op, hba->rpm_lvl, hba->spm_lvl, hba->uic_link_state, hba->curr_dev_pwr_mode, err); if (host->dbg_en) trace_ufs_qcom_resume(dev_name(hba->dev), pm_op, hba->rpm_lvl, hba->spm_lvl, hba->uic_link_state, hba->curr_dev_pwr_mode, err); return err; } static int ufs_qcom_get_bus_vote(struct ufs_qcom_host *host, const char *speed_mode) { struct device *dev = host->hba->dev; struct device_node *np = dev->of_node; int err; const char *key = "qcom,bus-vector-names"; if (!speed_mode) { err = -EINVAL; goto out; } if (host->bus_vote.is_max_bw_needed && !!strcmp(speed_mode, "MIN")) err = of_property_match_string(np, key, "MAX"); else err = of_property_match_string(np, key, speed_mode); out: if (err < 0) ufs_qcom_msg(ERR, dev, "%s: Invalid %s mode %d\n", __func__, speed_mode, err); return err; } static void ufs_qcom_get_speed_mode(struct ufs_pa_layer_attr *p, char *result) { int gear = max_t(u32, p->gear_rx, p->gear_tx); int lanes = max_t(u32, p->lane_rx, p->lane_tx); int pwr; /* default to PWM Gear 1, Lane 1 if power mode is not initialized */ if (!gear) gear = 1; if (!lanes) lanes = 1; if (!p->pwr_rx && !p->pwr_tx) { pwr = SLOWAUTO_MODE; snprintf(result, BUS_VECTOR_NAME_LEN, "MIN"); } else if (p->pwr_rx == FAST_MODE || p->pwr_rx == FASTAUTO_MODE || p->pwr_tx == FAST_MODE || p->pwr_tx == FASTAUTO_MODE) { pwr = FAST_MODE; snprintf(result, BUS_VECTOR_NAME_LEN, "%s_R%s_G%d_L%d", "HS", p->hs_rate == PA_HS_MODE_B ? "B" : "A", gear, lanes); } else { pwr = SLOW_MODE; snprintf(result, BUS_VECTOR_NAME_LEN, "%s_G%d_L%d", "PWM", gear, lanes); } } static int ufs_qcom_get_ib_ab(struct ufs_qcom_host *host, int index, struct qcom_bus_vectors *ufs_ddr_vec, struct qcom_bus_vectors *cpu_ufs_vec) { struct qcom_bus_path *usecase; if (!host->qbsd) return -EINVAL; if (index > host->qbsd->num_usecase) return -EINVAL; usecase = host->qbsd->usecase; /* * * usecase:0 usecase:0 * ufs->ddr cpu->ufs * |vec[0&1] | vec[2&3]| * +----+----+----+----+ * | ab | ib | ab | ib | * |----+----+----+----+ * . * . * . * usecase:n usecase:n * ufs->ddr cpu->ufs * |vec[0&1] | vec[2&3]| * +----+----+----+----+ * | ab | ib | ab | ib | * |----+----+----+----+ */ /* index refers to offset in usecase */ ufs_ddr_vec->ab = usecase[index].vec[0].ab; ufs_ddr_vec->ib = usecase[index].vec[0].ib; cpu_ufs_vec->ab = usecase[index].vec[1].ab; cpu_ufs_vec->ib = usecase[index].vec[1].ib; return 0; } static int __ufs_qcom_set_bus_vote(struct ufs_qcom_host *host, int vote) { int err = 0; struct qcom_bus_scale_data *d = host->qbsd; struct qcom_bus_vectors path0, path1; struct device *dev = host->hba->dev; err = ufs_qcom_get_ib_ab(host, vote, &path0, &path1); if (err) { ufs_qcom_msg(ERR, dev, "Error: failed (%d) to get ib/ab\n", err); return err; } ufs_qcom_msg(DBG, dev, "Setting vote: %d: ufs-ddr: ab: %llu ib: %llu\n", vote, path0.ab, path0.ib); err = icc_set_bw(d->ufs_ddr, path0.ab, path0.ib); if (err) { ufs_qcom_msg(ERR, dev, "Error: failed setting (%s) bus vote\n", err, UFS_DDR); return err; } ufs_qcom_msg(DBG, dev, "Setting: cpu-ufs: ab: %llu ib: %llu\n", path1.ab, path1.ib); err = icc_set_bw(d->cpu_ufs, path1.ab, path1.ib); if (err) { ufs_qcom_msg(ERR, dev, "Error: failed setting (%s) bus vote\n", err, CPU_UFS); return err; } host->bus_vote.curr_vote = vote; return err; } static int ufs_qcom_update_bus_bw_vote(struct ufs_qcom_host *host) { int vote; int err = 0; char mode[BUS_VECTOR_NAME_LEN]; ufs_qcom_get_speed_mode(&host->dev_req_params, mode); vote = ufs_qcom_get_bus_vote(host, mode); if (vote >= 0) err = __ufs_qcom_set_bus_vote(host, vote); else err = vote; if (err) ufs_qcom_msg(ERR, host->hba->dev, "%s: failed %d\n", __func__, err); else host->bus_vote.saved_vote = vote; return err; } static int ufs_qcom_set_bus_vote(struct ufs_hba *hba, bool on) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int vote, err; /* * In case ufs_qcom_init() is not yet done, simply ignore. * This ufs_qcom_set_bus_vote() shall be called from * ufs_qcom_init() after init is done. */ if (!host) return 0; if (on) { vote = host->bus_vote.saved_vote; if (vote == host->bus_vote.min_bw_vote) ufs_qcom_update_bus_bw_vote(host); } else { vote = host->bus_vote.min_bw_vote; } err = __ufs_qcom_set_bus_vote(host, vote); if (err) ufs_qcom_msg(ERR, hba->dev, "%s: set bus vote failed %d\n", __func__, err); return err; } static ssize_t show_ufs_to_mem_max_bus_bw(struct device *dev, struct device_attribute *attr, char *buf) { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); return scnprintf(buf, PAGE_SIZE, "%u\n", host->bus_vote.is_max_bw_needed); } static ssize_t store_ufs_to_mem_max_bus_bw(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); uint32_t value; if (!kstrtou32(buf, 0, &value)) { host->bus_vote.is_max_bw_needed = !!value; ufs_qcom_update_bus_bw_vote(host); } return count; } static struct qcom_bus_scale_data *ufs_qcom_get_bus_scale_data(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct device_node *of_node = dev->of_node; struct qcom_bus_scale_data *qsd; struct qcom_bus_path *usecase = NULL; int ret = 0, i = 0, j, num_paths, len; const uint32_t *vec_arr = NULL; bool mem_err = false; if (!pdev) { ufs_qcom_msg(ERR, dev, "Null platform device!\n"); return NULL; } qsd = devm_kzalloc(dev, sizeof(struct qcom_bus_scale_data), GFP_KERNEL); if (!qsd) return NULL; ret = of_property_read_string(of_node, "qcom,ufs-bus-bw,name", &qsd->name); if (ret) { ufs_qcom_msg(ERR, dev, "Error: (%d) Bus name missing!\n", ret); return NULL; } ret = of_property_read_u32(of_node, "qcom,ufs-bus-bw,num-cases", &qsd->num_usecase); if (ret) { ufs_qcom_msg(ERR, NULL, "Error: num-usecases not found\n"); goto err; } usecase = devm_kzalloc(dev, (sizeof(struct qcom_bus_path) * qsd->num_usecase), GFP_KERNEL); if (!usecase) return NULL; ret = of_property_read_u32(of_node, "qcom,ufs-bus-bw,num-paths", &num_paths); if (ret) { ufs_qcom_msg(ERR, NULL, "Error: num_paths not found\n"); return NULL; } vec_arr = of_get_property(of_node, "qcom,ufs-bus-bw,vectors-KBps", &len); if (vec_arr == NULL) { ufs_qcom_msg(ERR, NULL, "Error: Vector array not found\n"); return NULL; } for (i = 0; i < qsd->num_usecase; i++) { usecase[i].num_paths = num_paths; usecase[i].vec = devm_kzalloc(dev, num_paths * sizeof(struct qcom_bus_vectors), GFP_KERNEL); if (!usecase[i].vec) { mem_err = true; ufs_qcom_msg(ERR, dev, "Error: Failed to alloc mem for vectors\n"); goto err; } for (j = 0; j < num_paths; j++) { uint32_t tab; int idx = ((i * num_paths) + j) * 2; tab = vec_arr[idx]; usecase[i].vec[j].ab = ((tab & 0xff000000) >> 24) | ((tab & 0x00ff0000) >> 8) | ((tab & 0x0000ff00) << 8) | (tab << 24); tab = vec_arr[idx + 1]; usecase[i].vec[j].ib = ((tab & 0xff000000) >> 24) | ((tab & 0x00ff0000) >> 8) | ((tab & 0x0000ff00) << 8) | (tab << 24); ufs_qcom_msg(DBG, dev, "ab: %llu ib:%llu [i]: %d [j]: %d\n", usecase[i].vec[j].ab, usecase[i].vec[j].ib, i, j); } } qsd->usecase = usecase; return qsd; err: return NULL; } /** * ufs_qcom_enable_crash_on_err - read from DTS whether crash_on_err * should be enabled during boot. * @hba: per adapter instance */ static void ufs_qcom_enable_crash_on_err(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct device *dev = hba->dev; struct device_node *np = dev->of_node; if (!np) return; host->crash_on_err = of_property_read_bool(np, "qcom,enable_crash_on_err"); } static int ufs_qcom_bus_register(struct ufs_qcom_host *host) { int err = 0; struct device *dev = host->hba->dev; struct qcom_bus_scale_data *qsd; qsd = ufs_qcom_get_bus_scale_data(dev); if (!qsd) { ufs_qcom_msg(ERR, dev, "Failed: getting bus_scale data\n"); return 0; } host->qbsd = qsd; qsd->ufs_ddr = of_icc_get(dev, UFS_DDR); if (IS_ERR(qsd->ufs_ddr)) { ufs_qcom_msg(ERR, dev, "Error: (%d) failed getting %s path\n", PTR_ERR(qsd->ufs_ddr), UFS_DDR); return PTR_ERR(qsd->ufs_ddr); } qsd->cpu_ufs = of_icc_get(dev, CPU_UFS); if (IS_ERR(qsd->cpu_ufs)) { ufs_qcom_msg(ERR, dev, "Error: (%d) failed getting %s path\n", PTR_ERR(qsd->cpu_ufs), CPU_UFS); return PTR_ERR(qsd->cpu_ufs); } /* cache the vote index for minimum and maximum bandwidth */ host->bus_vote.min_bw_vote = ufs_qcom_get_bus_vote(host, "MIN"); host->bus_vote.max_bw_vote = ufs_qcom_get_bus_vote(host, "MAX"); host->bus_vote.max_bus_bw.show = show_ufs_to_mem_max_bus_bw; host->bus_vote.max_bus_bw.store = store_ufs_to_mem_max_bus_bw; sysfs_attr_init(&host->bus_vote.max_bus_bw.attr); host->bus_vote.max_bus_bw.attr.name = "max_bus_bw"; host->bus_vote.max_bus_bw.attr.mode = 0644; err = device_create_file(dev, &host->bus_vote.max_bus_bw); if (err) ufs_qcom_msg(ERR, dev, "Error: (%d) Failed to create sysfs entries\n", err); return 0; } static void ufs_qcom_dev_ref_clk_ctrl(struct ufs_qcom_host *host, bool enable) { if (host->dev_ref_clk_ctrl_mmio && (enable ^ host->is_dev_ref_clk_enabled)) { u32 temp = readl_relaxed(host->dev_ref_clk_ctrl_mmio); if (enable) temp |= host->dev_ref_clk_en_mask; else temp &= ~host->dev_ref_clk_en_mask; /* * If we are here to disable this clock it might be immediately * after entering into hibern8 in which case we need to make * sure that device ref_clk is active for specific time after * hibern8 enter. */ if (!enable) { unsigned long gating_wait; gating_wait = host->hba->dev_info.clk_gating_wait_us; if (!gating_wait) { udelay(1); } else { /* * bRefClkGatingWaitTime defines the minimum * time for which the reference clock is * required by device during transition from * HS-MODE to LS-MODE or HIBERN8 state. Give it * more delay to be on the safe side. */ gating_wait += 10; usleep_range(gating_wait, gating_wait + 10); } } writel_relaxed(temp, host->dev_ref_clk_ctrl_mmio); /* * Make sure the write to ref_clk reaches the destination and * not stored in a Write Buffer (WB). */ readl(host->dev_ref_clk_ctrl_mmio); /* * If we call hibern8 exit after this, we need to make sure that * device ref_clk is stable for a given time before the hibern8 * exit command. */ if (enable) usleep_range(50, 60); host->is_dev_ref_clk_enabled = enable; } } static int ufs_qcom_pwr_change_notify(struct ufs_hba *hba, enum ufs_notify_change_status status, struct ufs_pa_layer_attr *dev_max_params, struct ufs_pa_layer_attr *dev_req_params) { u32 val; struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct phy *phy = host->generic_phy; struct ufs_qcom_dev_params ufs_qcom_cap; int ret = 0; if (!dev_req_params) { ufs_qcom_msg(ERR, NULL, "%s: incoming dev_req_params is NULL\n", __func__); ret = -EINVAL; goto out; } switch (status) { case PRE_CHANGE: ufs_qcom_cap.hs_rx_gear = host->limit_rx_hs_gear; ufs_qcom_cap.hs_tx_gear = host->limit_tx_hs_gear; ufs_qcom_cap.pwm_tx_gear = host->limit_tx_pwm_gear; ufs_qcom_cap.pwm_rx_gear = host->limit_rx_pwm_gear; ufs_qcom_cap.tx_lanes = hba->lanes_per_direction; ufs_qcom_cap.rx_lanes = hba->lanes_per_direction; ufs_qcom_cap.rx_pwr_pwm = UFS_QCOM_LIMIT_RX_PWR_PWM; ufs_qcom_cap.tx_pwr_pwm = UFS_QCOM_LIMIT_TX_PWR_PWM; ufs_qcom_cap.rx_pwr_hs = UFS_QCOM_LIMIT_RX_PWR_HS; ufs_qcom_cap.tx_pwr_hs = UFS_QCOM_LIMIT_TX_PWR_HS; ufs_qcom_cap.hs_rate = host->limit_rate; ufs_qcom_cap.desired_working_mode = UFS_QCOM_LIMIT_DESIRED_MODE; ret = ufs_qcom_get_pwr_dev_param(&ufs_qcom_cap, dev_max_params, dev_req_params); if (ret) { ufs_qcom_msg(ERR, NULL, "%s: failed to determine capabilities\n", __func__); goto out; } /* enable the device ref clock before changing to HS mode */ if (!ufshcd_is_hs_mode(&hba->pwr_info) && ufshcd_is_hs_mode(dev_req_params)) ufs_qcom_dev_ref_clk_ctrl(host, true); if (host->hw_ver.major >= 0x4) { if (dev_req_params->gear_tx >= UFS_HS_G4) { /* INITIAL ADAPT */ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE), PA_INITIAL_ADAPT); } else { /* NO ADAPT */ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE), PA_NO_ADAPT); } } break; case POST_CHANGE: if (ufs_qcom_cfg_timers(hba, dev_req_params->gear_rx, dev_req_params->pwr_rx, dev_req_params->hs_rate, false)) { ufs_qcom_msg(ERR, hba->dev, "%s: ufs_qcom_cfg_timers() failed\n", __func__); /* * we return error code at the end of the routine, * but continue to configure UFS_PHY_TX_LANE_ENABLE * and bus voting as usual */ ret = -EINVAL; } val = ~(MAX_U32 << dev_req_params->lane_tx); ufs_qcom_phy_set_tx_lane_enable(phy, val); /* cache the power mode parameters to use internally */ memcpy(&host->dev_req_params, dev_req_params, sizeof(*dev_req_params)); ufs_qcom_update_bus_bw_vote(host); /* disable the device ref clock if entered PWM mode */ if (ufshcd_is_hs_mode(&hba->pwr_info) && !ufshcd_is_hs_mode(dev_req_params)) ufs_qcom_dev_ref_clk_ctrl(host, false); /* update BER threshold depends on gear mode */ if (!override_ber_threshold && !ret) ber_threshold = ber_table[dev_req_params->gear_rx].ber_threshold; break; default: ret = -EINVAL; break; } out: if (dev_req_params) { ufs_qcom_log_str(host, "@,%d,%d,%d,%d,%d\n", status, dev_req_params->gear_rx, dev_req_params->pwr_rx, dev_req_params->hs_rate, ret); if (host->dbg_en) trace_ufs_qcom_pwr_change_notify(dev_name(hba->dev), status, dev_req_params->gear_rx, dev_req_params->pwr_rx, dev_req_params->hs_rate, ret); } else { ufs_qcom_log_str(host, "@,%d,%d,%d,%d,%d\n", status, 0, 0, 0, ret); if (host->dbg_en) trace_ufs_qcom_pwr_change_notify(dev_name(hba->dev), status, 0, 0, 0, ret); } return ret; } static int ufs_qcom_vdd_hba_reg_notifier(struct notifier_block *nb, unsigned long event, void *data) { struct ufs_qcom_host *host = container_of(nb, struct ufs_qcom_host, vdd_hba_reg_nb); switch (event) { case REGULATOR_EVENT_DISABLE: /* The flag will be cleared during h8 exit post change */ if (ufs_qcom_is_link_hibern8(host->hba) && (host->chosen_algo != STATIC_ALLOC_ALG1)) host->vdd_hba_pc = true; break; default: break; } return NOTIFY_OK; } static void ufs_qcom_hibern8_notify(struct ufs_hba *hba, enum uic_cmd_dme uic_cmd, enum ufs_notify_change_status status) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); /* Apply shared ICE WA */ if (uic_cmd == UIC_CMD_DME_HIBER_EXIT && status == POST_CHANGE && host->vdd_hba_pc) { WARN_ON(host->chosen_algo == STATIC_ALLOC_ALG1); host->vdd_hba_pc = false; ufshcd_writel(hba, 0x18, UFS_MEM_ICE); ufshcd_writel(hba, 0x0, UFS_MEM_ICE); } } static int ufs_qcom_quirk_host_pa_saveconfigtime(struct ufs_hba *hba) { int err; u32 pa_vs_config_reg1; err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_VS_CONFIG_REG1), &pa_vs_config_reg1); if (err) goto out; /* Allow extension of MSB bits of PA_SaveConfigTime attribute */ err = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_VS_CONFIG_REG1), (pa_vs_config_reg1 | (1 << 12))); out: return err; } static void ufs_qcom_override_pa_h8time(struct ufs_hba *hba) { int ret; u32 pa_h8time = 0; ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_HIBERN8TIME), &pa_h8time); if (ret) { ufs_qcom_msg(ERR, hba->dev, "Failed getting PA_HIBERN8TIME: %d\n", ret); return; } /* 1 implies 100 us */ ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), pa_h8time + 1); if (ret) ufs_qcom_msg(ERR, hba->dev, "Failed updating PA_HIBERN8TIME: %d\n", ret); } static inline bool ufshcd_is_valid_pm_lvl(enum ufs_pm_level lvl) { return lvl >= 0 && lvl < UFS_PM_LVL_MAX; } static void ufshcd_parse_pm_levels(struct ufs_hba *hba) { struct device *dev = hba->dev; struct device_node *np = dev->of_node; struct ufs_qcom_host *host = ufshcd_get_variant(hba); enum ufs_pm_level rpm_lvl = UFS_PM_LVL_MAX, spm_lvl = UFS_PM_LVL_MAX; if (!np) return; if (host->is_dt_pm_level_read) return; if (!of_property_read_u32(np, "rpm-level", &rpm_lvl) && ufshcd_is_valid_pm_lvl(rpm_lvl)) hba->rpm_lvl = rpm_lvl; if (!of_property_read_u32(np, "spm-level", &spm_lvl) && ufshcd_is_valid_pm_lvl(spm_lvl)) hba->spm_lvl = spm_lvl; host->is_dt_pm_level_read = true; } /* * ufs_qcom_parse_pbl_rst_workaround_flag - read bypass-pbl-rst-wa entry from DT */ static void ufs_qcom_parse_pbl_rst_workaround_flag(struct ufs_qcom_host *host) { struct device_node *np = host->hba->dev->of_node; const char *str = "qcom,bypass-pbl-rst-wa"; if (!np) return; host->bypass_pbl_rst_wa = of_property_read_bool(np, str); } static void ufs_qcom_override_pa_tx_hsg1_sync_len(struct ufs_hba *hba) { #define PA_TX_HSG1_SYNC_LENGTH 0x1552 int err; int sync_len_val = 0x4A; err = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TX_HSG1_SYNC_LENGTH), sync_len_val); if (err) ufs_qcom_msg(ERR, hba->dev, "Failed (%d) set PA_TX_HSG1_SYNC_LENGTH(%d)\n", err, sync_len_val); } static int ufs_qcom_apply_dev_quirks(struct ufs_hba *hba) { unsigned long flags; int err = 0; spin_lock_irqsave(hba->host->host_lock, flags); /* Set the rpm auto suspend delay to 3s */ hba->host->hostt->rpm_autosuspend_delay = UFS_QCOM_AUTO_SUSPEND_DELAY; /* Set the default auto-hiberate idle timer value to 5ms */ hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 5) | FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3); /* Set the clock gating delay to performance mode */ hba->clk_gating.delay_ms = UFS_QCOM_CLK_GATING_DELAY_MS_PERF; spin_unlock_irqrestore(hba->host->host_lock, flags); if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME) err = ufs_qcom_quirk_host_pa_saveconfigtime(hba); if (hba->dev_info.wmanufacturerid == UFS_VENDOR_WDC) hba->dev_quirks |= UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE; if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_HIBER8TIME) ufs_qcom_override_pa_h8time(hba); if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TX_HSG1_SYNC_LENGTH) ufs_qcom_override_pa_tx_hsg1_sync_len(hba); ufshcd_parse_pm_levels(hba); if (hba->dev_info.wmanufacturerid == UFS_VENDOR_MICRON) hba->dev_quirks |= UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM; return err; } static u32 ufs_qcom_get_ufs_hci_version(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); if (host->hw_ver.major == 0x1) return ufshci_version(1, 1); else return ufshci_version(2, 0); } /** * ufs_qcom_advertise_quirks - advertise the known QCOM UFS controller quirks * @hba: host controller instance * * QCOM UFS host controller might have some non standard behaviours (quirks) * than what is specified by UFSHCI specification. Advertise all such * quirks to standard UFS host controller driver so standard takes them into * account. */ static void ufs_qcom_advertise_quirks(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); if (host->hw_ver.major == 0x01) { hba->quirks |= UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS | UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP | UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE; if (host->hw_ver.minor == 0x0001 && host->hw_ver.step == 0x0001) hba->quirks |= UFSHCD_QUIRK_BROKEN_INTR_AGGR; hba->quirks |= UFSHCD_QUIRK_BROKEN_LCC; } if (host->hw_ver.major == 0x2) { hba->quirks |= UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION; if (!ufs_qcom_cap_qunipro(host)) /* Legacy UniPro mode still need following quirks */ hba->quirks |= (UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS | UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE | UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP); } if (host->disable_lpm) hba->quirks |= UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8; else hba->quirks |= UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL; #if IS_ENABLED(CONFIG_SCSI_UFS_CRYPTO_QTI) hba->quirks |= UFSHCD_QUIRK_CUSTOM_KEYSLOT_MANAGER; #endif } static void ufs_qcom_set_caps(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); if (!host->disable_lpm) { hba->caps |= UFSHCD_CAP_CLK_GATING | UFSHCD_CAP_HIBERN8_WITH_CLK_GATING | UFSHCD_CAP_CLK_SCALING | UFSHCD_CAP_WB_WITH_CLK_SCALING | UFSHCD_CAP_AUTO_BKOPS_SUSPEND; hba->caps |= UFSHCD_CAP_AGGR_POWER_COLLAPSE; } if (!host->disable_wb_support) hba->caps |= UFSHCD_CAP_WB_EN; hba->caps |= UFSHCD_CAP_CRYPTO; if (host->hw_ver.major >= 0x2) host->caps = UFS_QCOM_CAP_QUNIPRO | UFS_QCOM_CAP_RETAIN_SEC_CFG_AFTER_PWR_COLLAPSE; if (host->hw_ver.major >= 0x3) { host->caps |= UFS_QCOM_CAP_QUNIPRO_CLK_GATING; /* * The UFS PHY attached to v3.0.0 controller supports entering * deeper low power state of SVS2. This lets the controller * run at much lower clock frequencies for saving power. * Assuming this and any future revisions of the controller * support this capability. Need to revist this assumption if * any future platform with this core doesn't support the * capability, as there will be no benefit running at lower * frequencies then. */ host->caps |= UFS_QCOM_CAP_SVS2; } if (host->hw_ver.major >= 0x5) host->caps |= UFS_QCOM_CAP_SHARED_ICE; } static int ufs_qcom_unvote_qos_all(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct ufs_qcom_qos_req *ufs_qos_req = host->ufs_qos; struct qos_cpu_group *qcg; int err = 0, i; if (!host->ufs_qos) return 0; qcg = ufs_qos_req->qcg; for (i = 0; i < ufs_qos_req->num_groups; i++, qcg++) { flush_work(&qcg->vwork); if (!qcg->voted) continue; err = ufs_qcom_update_qos_constraints(qcg, QOS_MAX); if (err) ufs_qcom_msg(ERR, hba->dev, "Failed (%d) removing qos grp(%d)\n", err, i); } return err; } /** * ufs_qcom_setup_clocks - enables/disable clocks * @hba: host controller instance * @on: If true, enable clocks else disable them. * @status: PRE_CHANGE or POST_CHANGE notify * * Returns 0 on success, non-zero on failure. */ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on, enum ufs_notify_change_status status) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int err = 0; struct phy *phy; u32 mode; /* * In case ufs_qcom_init() is not yet done, simply ignore. * This ufs_qcom_setup_clocks() shall be called from * ufs_qcom_init() after init is done. */ if (!host) return 0; phy = host->generic_phy; switch (status) { case PRE_CHANGE: if (on) { err = ufs_qcom_set_bus_vote(hba, true); if (ufs_qcom_is_link_hibern8(hba)) { err = ufs_qcom_enable_lane_clks(host); if (err) { ufs_qcom_msg(ERR, hba->dev, "%s: enable lane clks failed, ret=%d\n", __func__, err); return err; } ufs_qcom_phy_set_src_clk_h8_exit(phy); } if (!host->ref_clki->enabled) { err = clk_prepare_enable(host->ref_clki->clk); if (!err) host->ref_clki->enabled = on; else ufs_qcom_msg(ERR, hba->dev, "%s: Fail dev-ref-clk enabled, ret=%d\n", __func__, err); } if (!host->core_unipro_clki->enabled) { err = clk_prepare_enable(host->core_unipro_clki->clk); if (!err) host->core_unipro_clki->enabled = on; else ufs_qcom_msg(ERR, hba->dev, "%s: Fail core-unipro-clk enabled, ret=%d\n", __func__, err); } } else { if (!ufs_qcom_is_link_active(hba)) { clk_disable_unprepare(host->core_unipro_clki->clk); host->core_unipro_clki->enabled = on; err = ufs_qcom_phy_power_off(hba); if (err) { ufs_qcom_msg(ERR, hba->dev, "%s: phy power off failed, ret=%d\n", __func__, err); return err; } ufs_qcom_dev_ref_clk_ctrl(host, false); clk_disable_unprepare(host->ref_clki->clk); host->ref_clki->enabled = on; } } break; case POST_CHANGE: if (!on) { if (ufs_qcom_is_link_hibern8(hba)) { ufs_qcom_phy_set_src_clk_h8_enter(phy); /* * As XO is set to the source of lane clocks, hence * disable lane clocks to unvote on XO and allow XO shutdown */ ufs_qcom_disable_lane_clks(host); } err = ufs_qcom_set_bus_vote(hba, false); if (err) return err; err = ufs_qcom_unvote_qos_all(hba); idle_start = ktime_get(); } else { err = ufs_qcom_phy_power_on(hba); if (err) { ufs_qcom_msg(ERR, hba->dev, "%s: phy power on failed, ret = %d\n", __func__, err); return err; } if (ufshcd_is_hs_mode(&hba->pwr_info)) ufs_qcom_dev_ref_clk_ctrl(host, true); for (mode = 0; mode < UFS_QCOM_BER_MODE_MAX; mode++) idle_time[mode] += ktime_to_ms(ktime_sub(ktime_get(), idle_start)); } if (!err) atomic_set(&host->clks_on, on); break; } ufs_qcom_log_str(host, "#,%d,%d,%d\n", status, on, err); if (host->dbg_en) trace_ufs_qcom_setup_clocks(dev_name(hba->dev), status, on, err); return err; } static int ufs_qcom_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) { struct ufs_qcom_host *host = rcdev_to_ufs_host(rcdev); /* Currently this code only knows about a single reset. */ WARN_ON(id); ufs_qcom_assert_reset(host->hba); /* provide 1ms delay to let the reset pulse propagate. */ usleep_range(1000, 1100); return 0; } static int ufs_qcom_reset_deassert(struct reset_controller_dev *rcdev, unsigned long id) { struct ufs_qcom_host *host = rcdev_to_ufs_host(rcdev); /* Currently this code only knows about a single reset. */ WARN_ON(id); ufs_qcom_deassert_reset(host->hba); /* * after reset deassertion, phy will need all ref clocks, * voltage, current to settle down before starting serdes. */ usleep_range(1000, 1100); return 0; } static const struct reset_control_ops ufs_qcom_reset_ops = { .assert = ufs_qcom_reset_assert, .deassert = ufs_qcom_reset_deassert, }; #ifndef MODULE static int __init get_android_boot_dev(char *str) { strscpy(android_boot_dev, str, ANDROID_BOOT_DEV_MAX); return 1; } __setup("androidboot.bootdevice=", get_android_boot_dev); #endif static int ufs_qcom_parse_reg_info(struct ufs_qcom_host *host, char *name, struct ufs_vreg **out_vreg) { int ret = 0; char prop_name[MAX_PROP_SIZE]; struct ufs_vreg *vreg = NULL; struct device *dev = host->hba->dev; struct device_node *np = dev->of_node; if (!np) { ufs_qcom_msg(ERR, dev, "%s: non DT initialization\n", __func__); goto out; } snprintf(prop_name, MAX_PROP_SIZE, "%s-supply", name); if (!of_parse_phandle(np, prop_name, 0)) { ufs_qcom_msg(INFO, dev, "%s: Unable to find %s regulator, assuming enabled\n", __func__, prop_name); ret = -ENODEV; goto out; } vreg = devm_kzalloc(dev, sizeof(*vreg), GFP_KERNEL); if (!vreg) return -ENOMEM; vreg->name = name; snprintf(prop_name, MAX_PROP_SIZE, "%s-max-microamp", name); ret = of_property_read_u32(np, prop_name, &vreg->max_uA); if (ret) { ufs_qcom_msg(ERR, dev, "%s: unable to find %s err %d\n", __func__, prop_name, ret); goto out; } vreg->reg = devm_regulator_get(dev, vreg->name); if (IS_ERR(vreg->reg)) { ret = PTR_ERR(vreg->reg); ufs_qcom_msg(ERR, dev, "%s: %s get failed, err=%d\n", __func__, vreg->name, ret); } snprintf(prop_name, MAX_PROP_SIZE, "%s-min-uV", name); ret = of_property_read_u32(np, prop_name, &vreg->min_uV); if (ret) { ufs_qcom_msg(DBG, dev, "%s: unable to find %s err %d, using default\n", __func__, prop_name, ret); if (!strcmp(name, "qcom,vddp-ref-clk")) vreg->min_uV = VDDP_REF_CLK_MIN_UV; else if (!strcmp(name, "qcom,vccq-parent")) vreg->min_uV = 0; ret = 0; } snprintf(prop_name, MAX_PROP_SIZE, "%s-max-uV", name); ret = of_property_read_u32(np, prop_name, &vreg->max_uV); if (ret) { ufs_qcom_msg(DBG, dev, "%s: unable to find %s err %d, using default\n", __func__, prop_name, ret); if (!strcmp(name, "qcom,vddp-ref-clk")) vreg->max_uV = VDDP_REF_CLK_MAX_UV; else if (!strcmp(name, "qcom,vccq-parent")) vreg->max_uV = 0; ret = 0; } out: if (!ret) *out_vreg = vreg; return ret; } static void ufs_qcom_save_host_ptr(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int id; if (!hba->dev->of_node) return; /* Extract platform data */ id = of_alias_get_id(hba->dev->of_node, "ufshc"); if (id <= 0) ufs_qcom_msg(ERR, hba->dev, "Failed to get host index %d\n", id); else if (id <= MAX_UFS_QCOM_HOSTS) ufs_qcom_hosts[id - 1] = host; else ufs_qcom_msg(ERR, hba->dev, "invalid host index %d\n", id); } /** * ufs_qcom_query_ioctl - perform user read queries * @hba: per-adapter instance * @lun: used for lun specific queries * @buffer: user space buffer for reading and submitting query data and params * @return: 0 for success negative error code otherwise * * Expected/Submitted buffer structure is struct ufs_ioctl_query_data. * It will read the opcode, idn and buf_length parameters, and, put the * response in the buffer field while updating the used size in buf_length. */ static int ufs_qcom_query_ioctl(struct ufs_hba *hba, u8 lun, void __user *buffer) { struct ufs_ioctl_query_data *ioctl_data; int err = 0; int length = 0; void *data_ptr; bool flag; u32 att; u8 index; u8 *desc = NULL; ioctl_data = kzalloc(sizeof(*ioctl_data), GFP_KERNEL); if (!ioctl_data) { err = -ENOMEM; goto out; } /* extract params from user buffer */ err = copy_from_user(ioctl_data, buffer, sizeof(struct ufs_ioctl_query_data)); if (err) { ufs_qcom_msg(ERR, hba->dev, "%s: Failed copying buffer from user, err %d\n", __func__, err); goto out_release_mem; } #if defined(CONFIG_UFSFEATURE) if (ufsf_check_query(ioctl_data->opcode)) { err = ufsf_query_ioctl(ufs_qcom_get_ufsf(hba), lun, buffer, ioctl_data); goto out_release_mem; } #endif /* verify legal parameters & send query */ switch (ioctl_data->opcode) { case UPIU_QUERY_OPCODE_READ_DESC: switch (ioctl_data->idn) { case QUERY_DESC_IDN_DEVICE: case QUERY_DESC_IDN_CONFIGURATION: case QUERY_DESC_IDN_INTERCONNECT: case QUERY_DESC_IDN_GEOMETRY: case QUERY_DESC_IDN_POWER: index = 0; break; case QUERY_DESC_IDN_UNIT: if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun, 0)) { ufs_qcom_msg(ERR, hba->dev, "%s: No unit descriptor for lun 0x%x\n", __func__, lun); err = -EINVAL; goto out_release_mem; } index = lun; break; default: goto out_einval; } length = min_t(int, QUERY_DESC_MAX_SIZE, ioctl_data->buf_size); desc = kzalloc(length, GFP_KERNEL); if (!desc) { ufs_qcom_msg(ERR, hba->dev, "%s: Failed allocating %d bytes\n", __func__, length); err = -ENOMEM; goto out_release_mem; } err = ufshcd_query_descriptor_retry(hba, ioctl_data->opcode, ioctl_data->idn, index, 0, desc, &length); break; case UPIU_QUERY_OPCODE_READ_ATTR: switch (ioctl_data->idn) { case QUERY_ATTR_IDN_BOOT_LU_EN: case QUERY_ATTR_IDN_POWER_MODE: case QUERY_ATTR_IDN_ACTIVE_ICC_LVL: case QUERY_ATTR_IDN_OOO_DATA_EN: case QUERY_ATTR_IDN_BKOPS_STATUS: case QUERY_ATTR_IDN_PURGE_STATUS: case QUERY_ATTR_IDN_MAX_DATA_IN: case QUERY_ATTR_IDN_MAX_DATA_OUT: case QUERY_ATTR_IDN_REF_CLK_FREQ: case QUERY_ATTR_IDN_CONF_DESC_LOCK: case QUERY_ATTR_IDN_MAX_NUM_OF_RTT: case QUERY_ATTR_IDN_EE_CONTROL: case QUERY_ATTR_IDN_EE_STATUS: case QUERY_ATTR_IDN_SECONDS_PASSED: index = 0; break; case QUERY_ATTR_IDN_DYN_CAP_NEEDED: case QUERY_ATTR_IDN_CORR_PRG_BLK_NUM: index = lun; break; default: goto out_einval; } err = ufshcd_query_attr(hba, ioctl_data->opcode, ioctl_data->idn, index, 0, &att); break; case UPIU_QUERY_OPCODE_WRITE_ATTR: err = copy_from_user(&att, buffer + sizeof(struct ufs_ioctl_query_data), sizeof(u32)); if (err) { ufs_qcom_msg(ERR, hba->dev, "%s: Failed copying buffer from user, err %d\n", __func__, err); goto out_release_mem; } switch (ioctl_data->idn) { case QUERY_ATTR_IDN_BOOT_LU_EN: index = 0; if (!att) { ufs_qcom_msg(ERR, hba->dev, "%s: Illegal ufs query ioctl data, opcode 0x%x, idn 0x%x, att 0x%x\n", __func__, ioctl_data->opcode, (unsigned int)ioctl_data->idn, att); err = -EINVAL; goto out_release_mem; } break; default: goto out_einval; } err = ufshcd_query_attr(hba, ioctl_data->opcode, ioctl_data->idn, index, 0, &att); break; case UPIU_QUERY_OPCODE_READ_FLAG: switch (ioctl_data->idn) { case QUERY_FLAG_IDN_FDEVICEINIT: case QUERY_FLAG_IDN_PERMANENT_WPE: case QUERY_FLAG_IDN_PWR_ON_WPE: case QUERY_FLAG_IDN_BKOPS_EN: case QUERY_FLAG_IDN_PURGE_ENABLE: case QUERY_FLAG_IDN_FPHYRESOURCEREMOVAL: case QUERY_FLAG_IDN_BUSY_RTC: break; default: goto out_einval; } err = ufshcd_query_flag(hba, ioctl_data->opcode, ioctl_data->idn, 0, &flag); break; default: goto out_einval; } if (err) { ufs_qcom_msg(ERR, hba->dev, "%s: Query for idn %d failed\n", __func__, ioctl_data->idn); goto out_release_mem; } /* * copy response data * As we might end up reading less data than what is specified in * "ioctl_data->buf_size". So we are updating "ioctl_data-> * buf_size" to what exactly we have read. */ switch (ioctl_data->opcode) { case UPIU_QUERY_OPCODE_READ_DESC: ioctl_data->buf_size = min_t(int, ioctl_data->buf_size, length); data_ptr = desc; break; case UPIU_QUERY_OPCODE_READ_ATTR: ioctl_data->buf_size = sizeof(u32); data_ptr = &att; break; case UPIU_QUERY_OPCODE_READ_FLAG: ioctl_data->buf_size = 1; data_ptr = &flag; break; case UPIU_QUERY_OPCODE_WRITE_ATTR: goto out_release_mem; default: goto out_einval; } /* copy to user */ err = copy_to_user(buffer, ioctl_data, sizeof(struct ufs_ioctl_query_data)); if (err) ufs_qcom_msg(ERR, hba->dev, "%s: Failed copying back to user.\n", __func__); err = copy_to_user(buffer + sizeof(struct ufs_ioctl_query_data), data_ptr, ioctl_data->buf_size); if (err) ufs_qcom_msg(ERR, hba->dev, "%s: err %d copying back to user.\n", __func__, err); goto out_release_mem; out_einval: ufs_qcom_msg(ERR, hba->dev, "%s: illegal ufs query ioctl data, opcode 0x%x, idn 0x%x\n", __func__, ioctl_data->opcode, (unsigned int)ioctl_data->idn); err = -EINVAL; out_release_mem: kfree(ioctl_data); kfree(desc); out: return err; } /** * ufs_qcom_ioctl - ufs ioctl callback registered in scsi_host * @dev: scsi device required for per LUN queries * @cmd: command opcode * @buffer: user space buffer for transferring data * * Supported commands: * UFS_IOCTL_QUERY */ static int ufs_qcom_ioctl(struct scsi_device *dev, unsigned int cmd, void __user *buffer) { struct ufs_hba *hba = shost_priv(dev->host); int err = 0; BUG_ON(!hba); switch (cmd) { case UFS_IOCTL_QUERY: if (!buffer) { ufs_qcom_msg(ERR, hba->dev, "%s: User buffer is NULL!\n", __func__); return -EINVAL; } ufshcd_rpm_get_sync(hba); err = ufs_qcom_query_ioctl(hba, ufshcd_scsi_to_upiu_lun(dev->lun), buffer); ufshcd_rpm_put_sync(hba); break; default: err = -ENOIOCTLCMD; ufs_qcom_msg(DBG, hba->dev, "%s: Unsupported ioctl cmd %d\n", __func__, cmd); break; } return err; } static int tag_to_cpu(struct ufs_hba *hba, unsigned int tag) { struct ufshcd_lrb *lrbp = &hba->lrb[tag]; if (lrbp && lrbp->cmd && scsi_cmd_to_rq(lrbp->cmd)) return blk_mq_rq_cpu(scsi_cmd_to_rq(lrbp->cmd)); return -EINVAL; } static struct qos_cpu_group *cpu_to_group(struct ufs_qcom_qos_req *r, unsigned int cpu) { int i; struct qos_cpu_group *g = r->qcg; if (cpu > num_possible_cpus()) return NULL; for (i = 0; i < r->num_groups; i++, g++) { if (cpumask_test_cpu(cpu, &g->mask)) return &r->qcg[i]; } return NULL; } static int ufs_qcom_update_qos_constraints(struct qos_cpu_group *qcg, enum constraint type) { unsigned int vote; int cpu, err; struct dev_pm_qos_request *qos_req = qcg->qos_req; if (type == QOS_MAX) vote = S32_MAX; else vote = qcg->votes[type]; ufs_qcom_msg(DBG, qcg->host->hba->dev, "%s: qcg: 0x%08x | const: %d\n", __func__, qcg, type); if (qcg->curr_vote == vote) return 0; for_each_cpu(cpu, &qcg->mask) { err = dev_pm_qos_update_request(qos_req, vote); ufs_qcom_msg(DBG, qcg->host->hba->dev, "%s: vote: %d | cpu: %d | qos_req: 0x%08x\n", __func__, vote, cpu, qos_req); if (err < 0) return err; ++qos_req; } if (type == QOS_MAX) qcg->voted = false; else qcg->voted = true; qcg->curr_vote = vote; return 0; } static void ufs_qcom_qos(struct ufs_hba *hba, int tag, bool is_scsi_cmd) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct qos_cpu_group *qcg; int cpu; if (!host->ufs_qos) return; cpu = tag_to_cpu(hba, tag); if (cpu < 0) return; qcg = cpu_to_group(host->ufs_qos, cpu); if (!qcg) return; if (qcg->perf_core && !host->cpufreq_dis && !!atomic_read(&host->scale_up)) atomic_inc(&host->num_reqs_threshold); if (qcg->voted) { ufs_qcom_msg(DBG, qcg->host->hba->dev, "%s: qcg: 0x%08x | Mask: 0x%08x - Already voted - return\n", __func__, qcg, qcg->mask); return; } queue_work(host->ufs_qos->workq, &qcg->vwork); ufs_qcom_msg(DBG, hba->dev, "Queued QoS work- cpu: %d\n", cpu); } static void ufs_qcom_vote_work(struct work_struct *work) { int err; struct qos_cpu_group *qcg = container_of(work, struct qos_cpu_group, vwork); err = ufs_qcom_update_qos_constraints(qcg, QOS_PERF); if (err) ufs_qcom_msg(ERR, qcg->host->hba->dev, "%s: update qos - failed: %d\n", __func__, err); } static int ufs_qcom_setup_qos(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct device *dev = hba->dev; struct ufs_qcom_qos_req *qr = host->ufs_qos; struct qos_cpu_group *qcg = qr->qcg; struct cpufreq_policy *policy; unsigned int cpu; int i, err; for (i = 0; i < qr->num_groups; i++, qcg++) { qcg->qos_req = kcalloc(cpumask_weight(&qcg->mask), sizeof(struct dev_pm_qos_request), GFP_KERNEL); if (!qcg->qos_req) { err = -ENOMEM; if (!i) return err; goto free_mem; } ufs_qcom_msg(DBG, dev, "%s: qcg: 0x%08x | mask: 0x%08x | mask-wt: %d | qos_req: 0x%08x\n", __func__, qcg, qcg->mask, cpumask_weight(&qcg->mask), qcg->qos_req); err = add_group_qos(qcg, S32_MAX); if (err < 0) { ufs_qcom_msg(ERR, dev, "Fail (%d) add qos-req: grp-%d\n", err, i); if (!i) { kfree(qcg->qos_req); return err; } goto free_mem; } INIT_WORK(&qcg->vwork, ufs_qcom_vote_work); if (host->cpufreq_dis || qcg->perf_core) continue; /* Bump up the cpufreq of the non-perf group */ cpu = cpumask_first((const struct cpumask *)&qcg->mask); policy = cpufreq_cpu_get(cpu); if (!policy) { ufs_qcom_msg(ERR, dev, "Failed cpufreq policy,cpu=%d,mask=0x%08x\n", __func__, cpu, qcg->mask); host->cpufreq_dis = true; host->config_cpu = -1; continue; } host->config_cpu = cpu; host->min_cpu_scale_freq = policy->cpuinfo.min_freq; host->max_cpu_scale_freq = policy->cpuinfo.max_freq; cpufreq_cpu_put(policy); } if (!host->cpufreq_dis) { err = ufs_qcom_init_cpu_minfreq_req(host, host->config_cpu); if (err) { ufs_qcom_msg(ERR, dev, "Failed to register for freq_qos: %d\n", err); host->cpufreq_dis = true; } else { INIT_DELAYED_WORK(&host->fwork, ufs_qcom_cpufreq_dwork); } } qr->workq = create_singlethread_workqueue("qc_ufs_qos_swq"); if (qr->workq) return 0; err = -1; free_mem: while (i--) { kfree(qcg->qos_req); qcg--; } return err; } static void ufs_qcom_qos_init(struct ufs_hba *hba) { struct device *dev = hba->dev; struct device_node *np = dev->of_node; struct device_node *group_node; struct ufs_qcom_qos_req *qr; struct qos_cpu_group *qcg; int i, err; struct ufs_qcom_host *host = ufshcd_get_variant(hba); host->cpufreq_dis = true; qr = kzalloc(sizeof(*qr), GFP_KERNEL); if (!qr) return; host->ufs_qos = qr; qr->num_groups = of_get_available_child_count(np); ufs_qcom_msg(DBG, hba->dev, "num-groups: %d\n", qr->num_groups); if (!qr->num_groups) { ufs_qcom_msg(ERR, dev, "QoS groups undefined\n"); kfree(qr); host->ufs_qos = NULL; return; } qcg = kzalloc(sizeof(*qcg) * qr->num_groups, GFP_KERNEL); if (!qcg) { kfree(qr); host->ufs_qos = NULL; return; } qr->qcg = qcg; for_each_available_child_of_node(np, group_node) { /* * Due to Logical contiguous CPU numbering, one to one mapping * between physical and logical cpu is no more applicable. * Hence we don't need to pass the qos mask from the device tree * and instead need to populate the mask dynamically * using available kernel API. */ if (of_property_read_bool(group_node, "perf")) { qcg->perf_core = true; qcg->mask.bits[0] = host->gold_mask.bits[0] | host->gold_prime_mask.bits[0]; host->cpufreq_dis = false; } else { qcg->perf_core = false; qcg->mask.bits[0] = host->silver_mask.bits[0]; } err = of_property_count_u32_elems(group_node, "vote"); if (err <= 0) { ufs_qcom_msg(ERR, dev, "1 vote is needed, bailing out: %d\n", err); goto out_err; } qcg->votes = kmalloc(sizeof(*qcg->votes) * err, GFP_KERNEL); if (!qcg->votes) goto out_err; for (i = 0; i < err; i++) { if (of_property_read_u32_index(group_node, "vote", i, &qcg->votes[i])) goto out_vote_err; } ufs_qcom_msg(DBG, dev, "%s: qcg: 0x%08x\n", __func__, qcg); qcg->host = host; ++qcg; } if (ufs_qcom_setup_qos(hba)) goto out_vote_err; return; out_vote_err: for (i = 0, qcg = qr->qcg; i < qr->num_groups; i++, qcg++) kfree(qcg->votes); out_err: kfree(qr->qcg); kfree(qr); host->ufs_qos = NULL; } /** * ufs_qcom_populate_available_cpus - Populate all the available cpu masks - * Silver, gold and gold prime. * @hba: per adapter instance */ static void ufs_qcom_populate_available_cpus(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int cid_cpu[MAX_NUM_CLUSTERS] = {-1, -1, -1}; int cid = -1; int prev_cid = -1; int cpu; /* * Due to Logical contiguous CPU numbering, one to one mapping * between physical and logical cpu is no more applicable. * Hence we are not passing cpu mask from the device tree. * Hence populate the cpu mask dynamically as below. */ for_each_cpu(cpu, cpu_possible_mask) { cid = topology_physical_package_id(cpu); if (cid != prev_cid) { cid_cpu[cid] = cpu; prev_cid = cid; } } /* * perf_mask which is needed for dynamic irq_affinity feature is updated * with either gold or gold_prime depending on the availability of the core. */ if (cid_cpu[SILVER_CORE] != -1) host->silver_mask.bits[0] = topology_core_cpumask(cid_cpu[SILVER_CORE])->bits[0]; if (cid_cpu[GOLD_CORE] != -1) { host->gold_mask.bits[0] = topology_core_cpumask(cid_cpu[GOLD_CORE])->bits[0]; host->perf_mask.bits[0] = host->gold_mask.bits[0]; } if (cid_cpu[GOLD_PRIME_CORE] != -1) { host->gold_prime_mask.bits[0] = topology_core_cpumask(cid_cpu[GOLD_PRIME_CORE])->bits[0]; host->perf_mask.bits[0] = host->gold_prime_mask.bits[0]; } } static void ufs_qcom_parse_irq_affinity(struct ufs_hba *hba) { struct device *dev = hba->dev; struct device_node *np = dev->of_node; struct ufs_qcom_host *host = ufshcd_get_variant(hba); int err; if (np) { /* check for dynamic irq affinity feature support */ err = of_property_read_bool(np, "qcom,dynamic-irq-affinity"); if (!err) { ufs_qcom_msg(DBG, host->hba->dev, "dynamic irq affinity not supported\n"); return; } } /* If perf mask is available then only enable dynamic irq affinity feature */ if (host->perf_mask.bits[0]) host->irq_affinity_support = true; } static void ufs_qcom_parse_pm_level(struct ufs_hba *hba) { struct device *dev = hba->dev; struct device_node *np = dev->of_node; struct ufs_qcom_host *host = ufshcd_get_variant(hba); if (np) { if (of_property_read_u32(np, "rpm-level", &hba->rpm_lvl)) hba->rpm_lvl = -1; if (of_property_read_u32(np, "spm-level", &hba->spm_lvl)) hba->spm_lvl = -1; if (of_property_read_bool(np, "set-ds-spm-level")) host->set_ds_spm_level = true; } } /* Returns the max mitigation level supported */ static int ufs_qcom_get_max_therm_state(struct thermal_cooling_device *tcd, unsigned long *data) { *data = UFS_QCOM_LVL_MAX_THERM; return 0; } /* Returns the current mitigation level */ static int ufs_qcom_get_cur_therm_state(struct thermal_cooling_device *tcd, unsigned long *data) { struct ufs_hba *hba = dev_get_drvdata(tcd->devdata); struct ufs_qcom_host *host = ufshcd_get_variant(hba); *data = host->uqt.curr_state; return 0; } /* Convert microseconds to Auto-Hibernate Idle Timer register value */ static u32 ufs_qcom_us_to_ahit(unsigned int timer) { unsigned int scale; for (scale = 0; timer > UFSHCI_AHIBERN8_TIMER_MASK; ++scale) timer /= UFSHCI_AHIBERN8_SCALE_FACTOR; return FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, timer) | FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, scale); } /* Sets the mitigation level to requested level */ static int ufs_qcom_set_cur_therm_state(struct thermal_cooling_device *tcd, unsigned long data) { struct ufs_hba *hba = dev_get_drvdata(tcd->devdata); struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct scsi_device *sdev; if (data == host->uqt.curr_state) return 0; switch (data) { case UFS_QCOM_LVL_NO_THERM: ufs_qcom_msg(WARN, tcd->devdata, "UFS host thermal mitigation stops\n"); if (host->irq_affinity_support) atomic_set(&host->therm_mitigation, 0); /* Set the default auto-hiberate idle timer to 5 ms */ ufshcd_auto_hibern8_update(hba, ufs_qcom_us_to_ahit(5000)); /* Set the default auto suspend delay to 3000 ms */ shost_for_each_device(sdev, hba->host) pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev, UFS_QCOM_AUTO_SUSPEND_DELAY); break; case UFS_QCOM_LVL_AGGR_THERM: case UFS_QCOM_LVL_MAX_THERM: ufs_qcom_msg(WARN, tcd->devdata, "Going into UFS host thermal mitigation state, performance may be impacted before UFS host thermal mitigation stops\n"); if (host->irq_affinity_support) { /* Stop setting hi-pri to requests and set irq affinity to default value */ atomic_set(&host->therm_mitigation, 1); cancel_dwork_unvote_cpufreq(hba); ufs_qcom_toggle_pri_affinity(hba, false); } /* Set the default auto-hiberate idle timer to 1 ms */ ufshcd_auto_hibern8_update(hba, ufs_qcom_us_to_ahit(1000)); /* Set the default auto suspend delay to 100 ms */ shost_for_each_device(sdev, hba->host) pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev, 100); break; default: ufs_qcom_msg(ERR, tcd->devdata, "Invalid UFS thermal state (%d)\n", data); return -EINVAL; } host->uqt.curr_state = data; return 0; } struct thermal_cooling_device_ops ufs_thermal_ops = { .get_max_state = ufs_qcom_get_max_therm_state, .get_cur_state = ufs_qcom_get_cur_therm_state, .set_cur_state = ufs_qcom_set_cur_therm_state, }; /* Static Algorithm */ static int ufs_qcom_config_alg1(struct ufs_hba *hba) { int ret; unsigned int val, rx_aes; struct ufs_qcom_host *host = ufshcd_get_variant(hba); unsigned int num_aes_cores; ret = of_property_read_u32(host->np, "rx-alloc-percent", &val); if (ret < 0) return ret; num_aes_cores = ufshcd_readl(hba, REG_UFS_MEM_ICE_NUM_AES_CORES); ufshcd_writel(hba, STATIC_ALLOC_ALG1, REG_UFS_MEM_SHARED_ICE_CONFIG); /* * DTS specifies the percent allocation to rx stream * Calculation - * Num Tx stream = N_TOT - (N_TOT * percent of rx stream allocation) */ rx_aes = DIV_ROUND_CLOSEST(num_aes_cores * val, 100); val = rx_aes | ((num_aes_cores - rx_aes) << 8); ufshcd_writel(hba, val, REG_UFS_MEM_SHARED_ICE_ALG1_NUM_CORE); return 0; } /* Floor based algorithm */ static int ufs_qcom_config_alg2(struct ufs_hba *hba) { int i, ret; unsigned int reg = REG_UFS_MEM_SHARED_ICE_ALG2_NUM_CORE_0; /* 6 values for each group, refer struct shared_ice_alg2_config */ unsigned int override_val[6]; struct ufs_qcom_host *host = ufshcd_get_variant(hba); char name[8]; memset(name, 0, sizeof(name)); ufshcd_writel(hba, FLOOR_BASED_ALG2, REG_UFS_MEM_SHARED_ICE_CONFIG); for (i = 0; i < ARRAY_SIZE(alg2_config); i++) { int core = 0, task = 0; if (host->np) { snprintf(name, sizeof(name), "%s%d", "g", i); ret = of_property_read_variable_u32_array(host->np, name, override_val, 6, 6); /* Some/All parameters may be overwritten */ if (ret > 0) __get_alg2_grp_params(override_val, &core, &task); else get_alg2_grp_params(i, &core, &task); } else { get_alg2_grp_params(i, &core, &task); } /* Num Core and Num task are contiguous & configured for a group together */ ufshcd_writel(hba, core, reg); reg += 4; ufshcd_writel(hba, task, reg); reg += 4; } return 0; } /* Instantaneous algorithm */ static int ufs_qcom_config_alg3(struct ufs_hba *hba) { unsigned int val[4]; int ret; unsigned int config; struct ufs_qcom_host *host = ufshcd_get_variant(hba); ret = of_property_read_variable_u32_array(host->np, "num-core", val, 4, 4); if (ret < 0) return ret; ufshcd_writel(hba, INSTANTANEOUS_ALG3, REG_UFS_MEM_SHARED_ICE_CONFIG); config = val[0] | (val[1] << 8) | (val[2] << 16) | (val[3] << 24); ufshcd_writel(hba, config, REG_UFS_MEM_SHARED_ICE_ALG3_NUM_CORE); return 0; } static int ufs_qcom_parse_shared_ice_config(struct ufs_hba *hba) { struct device_node *np; int ret; const char *alg_name; struct ufs_qcom_host *host = ufshcd_get_variant(hba); np = of_parse_phandle(hba->dev->of_node, "shared-ice-cfg", 0); if (!np) return -ENOENT; /* Only 1 algo can be enabled, pick the first */ host->np = of_get_next_available_child(np, NULL); if (!host->np) { ufs_qcom_msg(ERR, hba->dev, "Resort to default alg2\n"); /* No overrides, use floor based as default */ host->chosen_algo = FLOOR_BASED_ALG2; return 0; } ret = of_property_read_string(host->np, "alg-name", &alg_name); if (ret < 0) return ret; if (!strcmp(alg_name, "alg1")) host->chosen_algo = STATIC_ALLOC_ALG1; else if (!strcmp(alg_name, "alg2")) host->chosen_algo = FLOOR_BASED_ALG2; else if (!strcmp(alg_name, "alg3")) host->chosen_algo = INSTANTANEOUS_ALG3; else /* Absurd condition */ return -ENODATA; return ret; } static int ufs_qcom_config_shared_ice(struct ufs_qcom_host *host) { if (!is_shared_ice_supported(host)) return 0; switch (host->chosen_algo) { case STATIC_ALLOC_ALG1: return ufs_qcom_config_alg1(host->hba); case FLOOR_BASED_ALG2: return ufs_qcom_config_alg2(host->hba); case INSTANTANEOUS_ALG3: return ufs_qcom_config_alg3(host->hba); default: ufs_qcom_msg(ERR, host->hba->dev, "Unknown shared ICE algo (%d)\n", host->chosen_algo); } return -EINVAL; } static int ufs_qcom_shared_ice_init(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); if (!is_shared_ice_supported(host)) return 0; /* Shared ICE is enabled by default */ return ufs_qcom_parse_shared_ice_config(hba); } static int ufs_qcom_populate_ref_clk_ctrl(struct ufs_hba *hba) { struct device *dev = hba->dev; struct platform_device *pdev = to_platform_device(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct resource *res; /* * for newer controllers, device reference clock control bit has * moved inside UFS controller register address space itself. */ if (host->hw_ver.major >= 0x02) { host->dev_ref_clk_ctrl_mmio = hba->mmio_base + REG_UFS_CFG1; host->dev_ref_clk_en_mask = BIT(26); return 0; } /* "dev_ref_clk_ctrl_mem" is optional resource */ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dev_ref_clk_ctrl_mem"); if (res) { host->dev_ref_clk_ctrl_mmio = devm_ioremap_resource(dev, res); if (IS_ERR(host->dev_ref_clk_ctrl_mmio)) { dev_warn(dev, "%s: could not map dev_ref_clk_ctrl_mmio, err %ld\n", __func__, PTR_ERR(host->dev_ref_clk_ctrl_mmio)); host->dev_ref_clk_ctrl_mmio = NULL; } host->dev_ref_clk_en_mask = BIT(5); } return 0; } static void ufs_qcom_setup_max_hs_gear(struct ufs_qcom_host *host) { u32 param0; if (host->hw_ver.major == 0x1) { /* * HS-G3 operations may not reliably work on legacy QCOM UFS * host controller hardware even though capability exchange * during link startup phase may end up negotiating maximum * supported gear as G3. Hence, downgrade the maximum supported * gear to HS-G2. */ host->max_hs_gear = UFS_HS_G2; } else if (host->hw_ver.major < 0x4) { host->max_hs_gear = UFS_HS_G3; } else { param0 = ufshcd_readl(host->hba, REG_UFS_PARAM0); host->max_hs_gear = UFS_QCOM_MAX_HS_GEAR(param0); } } static void ufs_qcom_register_minidump(uintptr_t vaddr, u64 size, const char *buf_name, u64 id) { struct md_region md_entry; int ret; if (!msm_minidump_enabled()) return; scnprintf(md_entry.name, sizeof(md_entry.name), "%s_%d", buf_name, id); md_entry.virt_addr = vaddr; md_entry.phys_addr = virt_to_phys((void *)vaddr); md_entry.size = size; ret = msm_minidump_add_region(&md_entry); if (ret < 0) { pr_err("Failed to register UFS buffer %s in Minidump ret %d\n", buf_name, ret); return; } } static int ufs_qcom_panic_handler(struct notifier_block *nb, unsigned long e, void *p) { struct ufs_qcom_host *host = container_of(nb, struct ufs_qcom_host, ufs_qcom_panic_nb); struct ufs_hba *hba = host->hba; dev_err(hba->dev, "......dumping ufs info .......\n"); dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state); dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n", hba->outstanding_reqs, hba->outstanding_tasks); dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n", hba->saved_err, hba->saved_uic_err); dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n", hba->curr_dev_pwr_mode, hba->uic_link_state); dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n", hba->pm_op_in_progress, hba->is_sys_suspended); dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state); dev_err(hba->dev, "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n", ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp), hba->ufs_stats.hibern8_exit_cnt); dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n", ktime_to_us(hba->ufs_stats.last_intr_ts), hba->ufs_stats.last_intr_status); dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n", hba->eh_flags, hba->req_abort_count); dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n", hba->ufs_version, hba->capabilities, hba->caps); dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks, hba->dev_quirks); dev_err(hba->dev, "[RX, TX]: gear=[%d, %d], lane[%d, %d], rate = %d\n", hba->pwr_info.gear_rx, hba->pwr_info.gear_tx, hba->pwr_info.lane_rx, hba->pwr_info.lane_tx, hba->pwr_info.hs_rate); dev_err(hba->dev, "UFS RPM level = %d\n", hba->rpm_lvl); dev_err(hba->dev, "UFS SPM level = %d\n", hba->spm_lvl); dev_err(hba->dev, "host_blocked=%d\n host_failed =%d\n Host self-block=%d\n", hba->host->host_blocked, hba->host->host_failed, hba->host->host_self_blocked); dev_err(hba->dev, "............. ufs dump complete ..........\n"); return NOTIFY_OK; } static void ufs_qcom_set_rate_a(struct ufs_qcom_host *host) { size_t len; u8 *data; struct nvmem_cell *nvmem_cell; nvmem_cell = nvmem_cell_get(host->hba->dev, "ufs_dev"); if (IS_ERR(nvmem_cell)) { dev_err(host->hba->dev, "(%s) Failed to get nvmem cell\n", __func__); return; } data = (u8 *)nvmem_cell_read(nvmem_cell, &len); if (IS_ERR(data)) { dev_err(host->hba->dev, "(%s) Failed to read from nvmem\n", __func__); goto cell_put; } /* * data equal to zero shows that ufs 2.x card is connected while * non-zero value shows that ufs 3.x card is connected */ if (*data) { host->limit_rate = PA_HS_MODE_A; dev_dbg(host->hba->dev, "UFS 3.x device is detected, Mode is set to RATE A\n"); } kfree(data); cell_put: nvmem_cell_put(nvmem_cell); } /** * ufs_qcom_init - bind phy with controller * @hba: host controller instance * * Binds PHY with controller and powers up PHY enabling clocks * and regulators. * * Returns -EPROBE_DEFER if binding fails, returns negative error * on phy power up failure and returns zero on success. */ static int ufs_qcom_init(struct ufs_hba *hba) { char type[5]; int err, host_id = 0; struct device *dev = hba->dev; struct ufs_qcom_host *host; struct ufs_qcom_thermal *ut; struct ufs_clk_info *clki; host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); if (!host) { err = -ENOMEM; ufs_qcom_msg(ERR, dev, "%s: no memory for qcom ufs host\n", __func__); goto out; } /* Make a two way bind between the qcom host and the hba */ host->hba = hba; ufshcd_set_variant(hba, host); ut = &host->uqt; #if defined(CONFIG_UFS_DBG) host->dbg_en = true; #endif ufs_qcom_enable_crash_on_err(hba); /* Setup the reset control of HCI */ host->core_reset = devm_reset_control_get(hba->dev, "rst"); if (IS_ERR(host->core_reset)) { err = PTR_ERR(host->core_reset); ufs_qcom_msg(WARN, dev, "Failed to get reset control %d\n", err); host->core_reset = NULL; err = 0; } /* Fire up the reset controller. Failure here is non-fatal. */ host->rcdev.of_node = dev->of_node; host->rcdev.ops = &ufs_qcom_reset_ops; host->rcdev.owner = dev->driver->owner; host->rcdev.nr_resets = 1; err = devm_reset_controller_register(dev, &host->rcdev); if (err) { ufs_qcom_msg(WARN, dev, "Failed to register reset controller\n"); err = 0; } /* * voting/devoting device ref_clk source is time consuming hence * skip devoting it during aggressive clock gating. This clock * will still be gated off during runtime suspend. */ host->generic_phy = devm_phy_get(dev, "ufsphy"); if (host->generic_phy == ERR_PTR(-EPROBE_DEFER)) { /* * UFS driver might be probed before the phy driver does. * In that case we would like to return EPROBE_DEFER code. */ err = -EPROBE_DEFER; ufs_qcom_msg(WARN, dev, "%s: required phy device. hasn't probed yet. err = %d\n", __func__, err); goto out_variant_clear; } else if (IS_ERR(host->generic_phy)) { if (has_acpi_companion(dev)) { host->generic_phy = NULL; } else { err = PTR_ERR(host->generic_phy); ufs_qcom_msg(ERR, dev, "%s: PHY get failed %d\n", __func__, err); goto out_variant_clear; } } host->device_reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); if (IS_ERR(host->device_reset)) { err = PTR_ERR(host->device_reset); if (err != -EPROBE_DEFER) ufs_qcom_msg(ERR, dev, "failed to acquire reset gpio: %d\n", err); goto out_variant_clear; } err = ufs_qcom_bus_register(host); if (err) goto out_variant_clear; ufs_qcom_get_controller_revision(hba, &host->hw_ver.major, &host->hw_ver.minor, &host->hw_ver.step); ufs_qcom_populate_ref_clk_ctrl(hba); ufs_qcom_setup_max_hs_gear(host); /* Register vdd_hba vreg callback */ host->vdd_hba_reg_nb.notifier_call = ufs_qcom_vdd_hba_reg_notifier; devm_regulator_register_notifier(hba->vreg_info.vdd_hba->reg, &host->vdd_hba_reg_nb); /* update phy revision information before calling phy_init() */ err = ufs_qcom_phy_save_controller_version(host->generic_phy, host->hw_ver.major, host->hw_ver.minor, host->hw_ver.step); if (err == -EPROBE_DEFER) { pr_err("%s: phy device probe is not completed yet\n", __func__); goto out_variant_clear; } err = ufs_qcom_parse_reg_info(host, "qcom,vddp-ref-clk", &host->vddp_ref_clk); err = phy_init(host->generic_phy); if (err) { ufs_qcom_msg(ERR, hba->dev, "%s: phy init failed, err %d\n", __func__, err); goto out_variant_clear; } mutex_init(&host->phy_mutex); if (host->vddp_ref_clk) { err = ufs_qcom_enable_vreg(dev, host->vddp_ref_clk); if (err) { ufs_qcom_msg(ERR, dev, "%s: failed enabling ref clk supply: %d\n", __func__, err); goto out_phy_exit; } } err = ufs_qcom_parse_reg_info(host, "qcom,vccq-parent", &host->vccq_parent); if (host->vccq_parent) { err = ufs_qcom_enable_vreg(dev, host->vccq_parent); if (err) { ufs_qcom_msg(ERR, dev, "%s: failed enable vccq-parent err=%d\n", __func__, err); goto out_disable_vddp; } } list_for_each_entry(clki, &hba->clk_list_head, list) { if (!strcmp(clki->name, "core_clk_unipro")) { clki->keep_link_active = true; host->core_unipro_clki = clki; } else if (!strcmp(clki->name, "ref_clk")) host->ref_clki = clki; } err = ufs_qcom_init_lane_clks(host); if (err) goto out_disable_vccq_parent; ufs_qcom_parse_pbl_rst_workaround_flag(host); ufs_qcom_parse_pm_level(hba); ufs_qcom_parse_limits(host); ufs_qcom_parse_g4_workaround_flag(host); ufs_qcom_parse_lpm(host); if (host->disable_lpm) pm_runtime_forbid(host->hba->dev); ufs_qcom_parse_wb(host); ufs_qcom_set_caps(hba); ufs_qcom_advertise_quirks(hba); err = ufs_qcom_shared_ice_init(hba); if (err) ufs_qcom_msg(ERR, hba->dev, "Shared ICE Init failed, ret=%d\n", err); err = ufs_qcom_ice_init(host); if (err) goto out_variant_clear; /* * Instantiate crypto capabilities for wrapped keys. * It is controlled by CONFIG_SCSI_UFS_CRYPTO_QTI. * If this is not defined, this API would return zero and * non-wrapped crypto capabilities will be instantiated. */ ufshcd_qti_hba_init_crypto_capabilities(hba); ufs_qcom_set_bus_vote(hba, true); /* enable the device ref clock for HS mode*/ if (ufshcd_is_hs_mode(&hba->pwr_info)) ufs_qcom_dev_ref_clk_ctrl(host, true); if (hba->dev->id < MAX_UFS_QCOM_HOSTS) ufs_qcom_hosts[hba->dev->id] = host; host->dbg_print_en |= UFS_QCOM_DEFAULT_DBG_PRINT_EN; ufs_qcom_get_default_testbus_cfg(host); err = ufs_qcom_testbus_config(host); if (err) { ufs_qcom_msg(WARN, dev, "%s: failed to configure the testbus %d\n", __func__, err); err = 0; } ufs_qcom_init_sysfs(hba); /* Provide SCSI host ioctl API */ hba->host->hostt->ioctl = (int (*)(struct scsi_device *, unsigned int, void __user *))ufs_qcom_ioctl; #ifdef CONFIG_COMPAT hba->host->hostt->compat_ioctl = (int (*)(struct scsi_device *, unsigned int, void __user *))ufs_qcom_ioctl; #endif /* * Based on host_id, pass the appropriate device type * to register thermal cooling device. */ host_id = of_alias_get_id(hba->dev->of_node, "ufshc"); if ((host_id < 0) || (host_id > MAX_UFS_QCOM_HOSTS)) { ufs_qcom_msg(ERR, hba->dev, "Failed to get host index %d\n", host_id); host_id = 1; } snprintf(type, sizeof(type), "ufs%d", host_id); ut->tcd = devm_thermal_of_cooling_device_register(dev, dev->of_node, type, dev, &ufs_thermal_ops); if (IS_ERR(ut->tcd)) ufs_qcom_msg(WARN, dev, "Thermal mitigation registration failed: %d\n", PTR_ERR(ut->tcd)); else host->uqt.curr_state = UFS_QCOM_LVL_NO_THERM; ufs_qcom_save_host_ptr(hba); ufs_qcom_populate_available_cpus(hba); ufs_qcom_qos_init(hba); ufs_qcom_parse_irq_affinity(hba); ufs_qcom_ber_mon_init(hba); host->ufs_ipc_log_ctx = ipc_log_context_create(UFS_QCOM_MAX_LOG_SZ, "ufs-qcom", 0); if (!host->ufs_ipc_log_ctx) ufs_qcom_msg(WARN, dev, "IPC Log init - failed\n"); /* register minidump */ if (msm_minidump_enabled()) { ufs_qcom_register_minidump((uintptr_t)host, sizeof(struct ufs_qcom_host), "UFS_QHOST", host_id); ufs_qcom_register_minidump((uintptr_t)hba, sizeof(struct ufs_hba), "UFS_HBA", host_id); ufs_qcom_register_minidump((uintptr_t)hba->host, sizeof(struct Scsi_Host), "UFS_SHOST", host_id); /* Register Panic handler to dump more information in case of kernel panic */ host->ufs_qcom_panic_nb.notifier_call = ufs_qcom_panic_handler; host->ufs_qcom_panic_nb.priority = INT_MAX - 1; err = atomic_notifier_chain_register(&panic_notifier_list, &host->ufs_qcom_panic_nb); if (err) ufs_qcom_msg(WARN, dev, "Fail to register UFS panic notifier\n"); } /* Provide SCSI host ioctl API */ hba->host->hostt->ioctl = (int (*)(struct scsi_device *, unsigned int, void __user *))ufs_qcom_ioctl; #ifdef CONFIG_COMPAT hba->host->hostt->compat_ioctl = (int (*)(struct scsi_device *, unsigned int, void __user *))ufs_qcom_ioctl; #endif #if defined(CONFIG_UFSFEATURE) ufsf_set_init_state(hba); #endif goto out; out_disable_vccq_parent: if (host->vccq_parent) ufs_qcom_disable_vreg(dev, host->vccq_parent); out_disable_vddp: if (host->vddp_ref_clk) ufs_qcom_disable_vreg(dev, host->vddp_ref_clk); out_phy_exit: phy_exit(host->generic_phy); out_variant_clear: ufshcd_set_variant(hba, NULL); out: return err; } static void ufs_qcom_exit(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); ufs_qcom_disable_lane_clks(host); ufs_qcom_phy_power_off(hba); phy_exit(host->generic_phy); } static int ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(struct ufs_hba *hba, u32 clk_1us_cycles, u32 clk_40ns_cycles) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int err; u32 core_clk_ctrl_reg, clk_cycles; u32 mask = DME_VS_CORE_CLK_CTRL_MAX_CORE_CLK_1US_CYCLES_MASK; u32 offset = 0; /* Bits mask and offset changed on UFS host controller V4.0.0 onwards */ if (host->hw_ver.major >= 4) { mask = DME_VS_CORE_CLK_CTRL_MAX_CORE_CLK_1US_CYCLES_MASK_V4; offset = DME_VS_CORE_CLK_CTRL_MAX_CORE_CLK_1US_CYCLES_OFFSET_V4; } if (clk_1us_cycles > mask) return -EINVAL; err = ufshcd_dme_get(hba, UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL), &core_clk_ctrl_reg); if (err) goto out; core_clk_ctrl_reg &= ~mask; core_clk_ctrl_reg |= clk_1us_cycles; core_clk_ctrl_reg <<= offset; /* Clear CORE_CLK_DIV_EN */ core_clk_ctrl_reg &= ~DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT; err = ufshcd_dme_set(hba, UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL), core_clk_ctrl_reg); /* UFS host controller V4.0.0 onwards needs to program * PA_VS_CORE_CLK_40NS_CYCLES attribute per programmed frequency of * unipro core clk of UFS host controller. */ if (!err && (host->hw_ver.major >= 4)) { if (clk_40ns_cycles > PA_VS_CORE_CLK_40NS_CYCLES_MASK) return -EINVAL; err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_VS_CORE_CLK_40NS_CYCLES), &clk_cycles); if (err) goto out; clk_cycles &= ~PA_VS_CORE_CLK_40NS_CYCLES_MASK; clk_cycles |= clk_40ns_cycles; err = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_VS_CORE_CLK_40NS_CYCLES), clk_cycles); } out: return err; } static int ufs_qcom_clk_scale_up_pre_change(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct ufs_pa_layer_attr *attr = &host->dev_req_params; int err = 0; if (!ufs_qcom_cap_qunipro(host)) goto out; if (attr) __ufs_qcom_cfg_timers(hba, attr->gear_rx, attr->pwr_rx, attr->hs_rate, false, true); err = ufs_qcom_set_dme_vs_core_clk_ctrl_max_freq_mode(hba); out: return err; } static int ufs_qcom_clk_scale_up_post_change(struct ufs_hba *hba) { unsigned long flags; spin_lock_irqsave(hba->host->host_lock, flags); hba->clk_gating.delay_ms = UFS_QCOM_CLK_GATING_DELAY_MS_PERF; spin_unlock_irqrestore(hba->host->host_lock, flags); return 0; } static int ufs_qcom_clk_scale_down_pre_change(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int err; u32 core_clk_ctrl_reg; if (!ufs_qcom_cap_qunipro(host)) return 0; err = ufshcd_dme_get(hba, UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL), &core_clk_ctrl_reg); /* make sure CORE_CLK_DIV_EN is cleared */ if (!err && (core_clk_ctrl_reg & DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT)) { core_clk_ctrl_reg &= ~DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT; err = ufshcd_dme_set(hba, UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL), core_clk_ctrl_reg); } return err; } static int ufs_qcom_clk_scale_down_post_change(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct ufs_pa_layer_attr *attr = &host->dev_req_params; int err = 0; struct ufs_clk_info *clki; struct list_head *head = &hba->clk_list_head; u32 curr_freq = 0; unsigned long flags; spin_lock_irqsave(hba->host->host_lock, flags); hba->clk_gating.delay_ms = UFS_QCOM_CLK_GATING_DELAY_MS_PWR_SAVE; spin_unlock_irqrestore(hba->host->host_lock, flags); if (!ufs_qcom_cap_qunipro(host)) return 0; if (attr) ufs_qcom_cfg_timers(hba, attr->gear_rx, attr->pwr_rx, attr->hs_rate, false); list_for_each_entry(clki, head, list) { if (!IS_ERR_OR_NULL(clki->clk) && (!strcmp(clki->name, "core_clk_unipro"))) { curr_freq = clk_get_rate(clki->clk); break; } } switch (curr_freq) { case 37500000: err = ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(hba, 38, 2); break; case 75000000: err = ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(hba, 75, 3); break; default: err = -EINVAL; break; } return err; } /** * Check controller status * Returns true if controller is active, false otherwise */ static bool is_hba_active(struct ufs_hba *hba) { return !!(ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE); } static int ufs_qcom_clk_scale_notify(struct ufs_hba *hba, bool scale_up, enum ufs_notify_change_status status) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct ufs_pa_layer_attr *dev_req_params = &host->dev_req_params; int err = 0; /* Check if controller is active to send commands, else commands will timeout */ if (!is_hba_active(hba)) goto out; if (status == PRE_CHANGE) { err = ufshcd_uic_hibern8_enter(hba); if (err) return err; if (scale_up) { err = ufs_qcom_clk_scale_up_pre_change(hba); if (!host->cpufreq_dis && !(atomic_read(&host->therm_mitigation))) { atomic_set(&host->num_reqs_threshold, 0); queue_delayed_work(host->ufs_qos->workq, &host->fwork, msecs_to_jiffies( UFS_QCOM_LOAD_MON_DLY_MS)); } } else { err = ufs_qcom_clk_scale_down_pre_change(hba); cancel_dwork_unvote_cpufreq(hba); } if (err) ufshcd_uic_hibern8_exit(hba); } else { if (scale_up) err = ufs_qcom_clk_scale_up_post_change(hba); else err = ufs_qcom_clk_scale_down_post_change(hba); if (err || !dev_req_params) { ufshcd_uic_hibern8_exit(hba); goto out; } ufs_qcom_cfg_timers(hba, dev_req_params->gear_rx, dev_req_params->pwr_rx, dev_req_params->hs_rate, false); ufs_qcom_update_bus_bw_vote(host); ufshcd_uic_hibern8_exit(hba); } if (!err) atomic_set(&host->scale_up, scale_up); out: if (scale_up) ufs_qcom_log_str(host, "^,%d,%d\n", status, err); else ufs_qcom_log_str(host, "v,%d,%d\n", status, err); if (host->dbg_en) trace_ufs_qcom_clk_scale_notify(dev_name(hba->dev), status, scale_up, err); return err; } /* * ufs_qcom_ber_mon_init - init BER monition history mode name and register domain list. */ static void ufs_qcom_ber_mon_init(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int m = UFS_QCOM_BER_MODE_G1_G4; INIT_LIST_HEAD(&host->regs_list_head); for (m = UFS_QCOM_BER_MODE_G1_G4; m < UFS_QCOM_BER_MODE_MAX; m++) { switch (m) { case UFS_QCOM_BER_MODE_G1_G4: host->ber_hist[m].name = "gear1-4"; break; case UFS_QCOM_BER_MODE_G5: host->ber_hist[m].name = "gear5"; break; default: break; } } } static void ufs_qcom_print_ber_hist(struct ufs_qcom_host *host) { struct ufs_qcom_ber_hist *h; int m = UFS_QCOM_BER_MODE_G1_G4; int i; for (m = UFS_QCOM_BER_MODE_G1_G4; m < UFS_QCOM_BER_MODE_MAX; m++) { h = &host->ber_hist[m]; for (i = 0; i < UFS_QCOM_EVT_LEN; i++) { int p = (i + h->pos) % UFS_QCOM_EVT_LEN; if (h->tstamp[p] == 0) continue; dev_err(host->hba->dev, "pa_err_%d=0x%x, err_code=0x%x, gear=%d, time=%lld us\n", p, h->uec_pa[p], h->err_code[p], h->gear[p], ktime_to_us(h->tstamp[p])); } dev_err(host->hba->dev, "total %llu PA error on %s mode\n", h->cnt, h->name); } } static bool ufs_qcom_cal_ber(struct ufs_qcom_host *host, struct ufs_qcom_ber_hist *h) { int idx_start, idx_end, i; s64 total_run_time = 0; s64 total_full_time = 0; u32 gear = h->gear[(h->pos + UFS_QCOM_EVT_LEN - 1) % UFS_QCOM_EVT_LEN]; if (!override_ber_threshold) ber_threshold = ber_table[gear].ber_threshold; if (!override_ber_duration) ber_mon_dur_ms = UFS_QCOM_BER_DUR_DEF_MS; dev_dbg(host->hba->dev, "ber=%d, dur_ms=%d, h->cnt=%d, pos=%d\n", ber_threshold, ber_mon_dur_ms, h->cnt, h->pos); /* return from here if total error count is not more than BER threshold */ if (h->cnt <= ber_threshold) return false; /* * BER threshold allows num (ber_threshold) of errors, and we have h->cnt errors, * go find the error event slot which records the #(h->cnt - ber_threshold) error. */ idx_end = (h->cnt - 1) % UFS_QCOM_EVT_LEN; idx_start = ((h->cnt - 1) - ber_threshold) % UFS_QCOM_EVT_LEN; /* * Calcuate the total valid running time * plus one here due to the first step in * following while loop is i--. */ i = idx_end + 1; do { i--; if (i < 0) i += UFS_QCOM_EVT_LEN; total_run_time += h->run_time[i]; total_full_time += h->full_time[i]; } while (i != idx_start); dev_dbg(host->hba->dev, "idx_start=%d, idx_end=%d, total_time=%lld ms, total_run_time=%lld ms\n", idx_start, idx_end, total_full_time, total_run_time); if (total_run_time < ber_mon_dur_ms) return true; return false; } static bool ufs_qcom_update_ber_event(struct ufs_qcom_host *host, u32 data, enum ufs_hs_gear_tag gear) { struct ufs_qcom_ber_hist *h; ktime_t t_prv; u32 mode = ber_table[gear].mode; bool rc = false; h = &host->ber_hist[mode]; h->uec_pa[h->pos] = data; /* For PHY errors, record PA error code */ if ((data & 0xFF) & ~UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) h->err_code[h->pos] = ufshcd_readl(host->hba, UFS_MEM_REG_PA_ERR_CODE); h->gear[h->pos] = gear; h->tstamp[h->pos] = ktime_get(); if (h->pos != 0) t_prv = h->tstamp[h->pos - 1]; else if (h->cnt == 0) t_prv = ms_to_ktime(0); else t_prv = h->tstamp[UFS_QCOM_EVT_LEN - 1]; /* Calcuate the valid running time since last PA error event */ h->full_time[h->pos] = ktime_to_ms(ktime_sub(h->tstamp[h->pos], t_prv)); h->run_time[h->pos] = h->full_time[h->pos] - idle_time[mode]; /* reset idle time for next error event. */ idle_time[mode] = 0; h->cnt++; h->pos = (h->pos + 1) % UFS_QCOM_EVT_LEN; /* don't need to calculate again if BER has already exceeded threshold */ if (!host->ber_th_exceeded) rc = ufs_qcom_cal_ber(host, h); return rc; } static void ufs_qcom_save_all_regs(struct ufs_qcom_host *host) { struct phy *phy = host->generic_phy; ufs_qcom_save_regs(host, 0, UFSHCI_REG_SPACE_SIZE, "host_regs "); ufs_qcom_save_regs(host, REG_UFS_SYS1CLK_1US, 16 * 4, "HCI Vendor Specific Registers "); ufs_qcom_save_regs(host, UFS_MEM_ICE, 29 * 4, "HCI Shared ICE Registers "); ufs_qcom_save_regs(host, UFS_TEST_BUS, 4, "UFS_TEST_BUS "); ufs_qcom_save_testbus(host); ufs_qcom_phy_dbg_register_save(phy); } static void ufs_qcom_event_notify(struct ufs_hba *hba, enum ufs_event_type evt, void *data) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct phy *phy = host->generic_phy; u32 reg = *(u32 *)data; bool ber_th_exceeded = false; bool evt_valid = true; switch (evt) { case UFS_EVT_PA_ERR: ber_th_exceeded = ufs_qcom_update_ber_event(host, reg, hba->pwr_info.gear_rx); if (ber_th_exceeded) { host->ber_th_exceeded = true; ufs_qcom_save_all_regs(host); ufs_qcom_print_ber_hist(host); if (crash_on_ber) { ufs_qcom_phy_dbg_register_dump(phy); BUG_ON(1); } } if (host->ber_th_exceeded) dev_warn_ratelimited(hba->dev, "Warning: UFS BER exceeds threshold !!!\n"); break; case UFS_EVT_DL_ERR: if (reg == UIC_DATA_LINK_LAYER_EC_PA_ERROR_IND_RECEIVED) evt_valid = false; break; #if defined(CONFIG_UFSFEATURE) case UFS_EVT_WL_SUSP_ERR: ufsf_resume(ufs_qcom_get_ufsf(hba), true); break; #endif default: break; } if (evt_valid) host->valid_evt_cnt[evt]++; } void ufs_qcom_print_hw_debug_reg_all(struct ufs_hba *hba, void *priv, void (*print_fn)(struct ufs_hba *hba, int offset, int num_regs, const char *str, void *priv)) { u32 reg; struct ufs_qcom_host *host; if (unlikely(!hba)) { ufs_qcom_msg(ERR, NULL, "%s: hba is NULL\n", __func__); return; } if (unlikely(!print_fn)) { ufs_qcom_msg(ERR, hba->dev, "%s: print_fn is NULL\n", __func__); return; } host = ufshcd_get_variant(hba); if (!(host->dbg_print_en & UFS_QCOM_DBG_PRINT_REGS_EN)) return; reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_REG_OCSC); print_fn(hba, reg, 44, "UFS_UFS_DBG_RD_REG_OCSC ", priv); reg = ufshcd_readl(hba, REG_UFS_CFG1); reg |= UTP_DBG_RAMS_EN; ufshcd_writel(hba, reg, REG_UFS_CFG1); reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_EDTL_RAM); print_fn(hba, reg, 32, "UFS_UFS_DBG_RD_EDTL_RAM ", priv); reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_DESC_RAM); print_fn(hba, reg, 128, "UFS_UFS_DBG_RD_DESC_RAM ", priv); reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_PRDT_RAM); print_fn(hba, reg, 64, "UFS_UFS_DBG_RD_PRDT_RAM ", priv); /* clear bit 17 - UTP_DBG_RAMS_EN */ ufshcd_rmwl(hba, UTP_DBG_RAMS_EN, 0, REG_UFS_CFG1); reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_UAWM); print_fn(hba, reg, 4, "UFS_DBG_RD_REG_UAWM ", priv); reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_UARM); print_fn(hba, reg, 4, "UFS_DBG_RD_REG_UARM ", priv); reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TXUC); print_fn(hba, reg, 48, "UFS_DBG_RD_REG_TXUC ", priv); reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_RXUC); print_fn(hba, reg, 27, "UFS_DBG_RD_REG_RXUC ", priv); reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_DFC); print_fn(hba, reg, 19, "UFS_DBG_RD_REG_DFC ", priv); reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TRLUT); print_fn(hba, reg, 34, "UFS_DBG_RD_REG_TRLUT ", priv); reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TMRLUT); print_fn(hba, reg, 9, "UFS_DBG_RD_REG_TMRLUT ", priv); } static void ufs_qcom_enable_test_bus(struct ufs_qcom_host *host) { if (host->dbg_print_en & UFS_QCOM_DBG_PRINT_TEST_BUS_EN) { ufshcd_rmwl(host->hba, UFS_REG_TEST_BUS_EN, UFS_REG_TEST_BUS_EN, REG_UFS_CFG1); ufshcd_rmwl(host->hba, TEST_BUS_EN, TEST_BUS_EN, REG_UFS_CFG1); } else { ufshcd_rmwl(host->hba, UFS_REG_TEST_BUS_EN, 0, REG_UFS_CFG1); ufshcd_rmwl(host->hba, TEST_BUS_EN, 0, REG_UFS_CFG1); } } static void ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host *host) { /* provide a legal default configuration */ host->testbus.select_major = TSTBUS_UNIPRO; host->testbus.select_minor = 37; } static bool ufs_qcom_testbus_cfg_is_ok(struct ufs_qcom_host *host) { if (host->testbus.select_major >= TSTBUS_MAX) { ufs_qcom_msg(ERR, host->hba->dev, "%s: UFS_CFG1[TEST_BUS_SEL} may not equal 0x%05X\n", __func__, host->testbus.select_major); return false; } return true; } /* * The caller of this function must make sure that the controller * is out of runtime suspend and appropriate clocks are enabled * before accessing. */ int ufs_qcom_testbus_config(struct ufs_qcom_host *host) { int reg; int offset; u32 mask = TEST_BUS_SUB_SEL_MASK; unsigned long flags; struct ufs_hba *hba; if (!host) return -EINVAL; hba = host->hba; if (in_task()) spin_lock_irqsave(hba->host->host_lock, flags); if (!ufs_qcom_testbus_cfg_is_ok(host)) return -EPERM; switch (host->testbus.select_major) { case TSTBUS_UAWM: reg = UFS_TEST_BUS_CTRL_0; offset = 24; break; case TSTBUS_UARM: reg = UFS_TEST_BUS_CTRL_0; offset = 16; break; case TSTBUS_TXUC: reg = UFS_TEST_BUS_CTRL_0; offset = 8; break; case TSTBUS_RXUC: reg = UFS_TEST_BUS_CTRL_0; offset = 0; break; case TSTBUS_DFC: reg = UFS_TEST_BUS_CTRL_1; offset = 24; break; case TSTBUS_TRLUT: reg = UFS_TEST_BUS_CTRL_1; offset = 16; break; case TSTBUS_TMRLUT: reg = UFS_TEST_BUS_CTRL_1; offset = 8; break; case TSTBUS_OCSC: reg = UFS_TEST_BUS_CTRL_1; offset = 0; break; case TSTBUS_WRAPPER: reg = UFS_TEST_BUS_CTRL_2; offset = 16; break; case TSTBUS_COMBINED: reg = UFS_TEST_BUS_CTRL_2; offset = 8; break; case TSTBUS_UTP_HCI: reg = UFS_TEST_BUS_CTRL_2; offset = 0; break; case TSTBUS_UNIPRO: reg = UFS_UNIPRO_CFG; offset = 20; mask = 0xFFF; break; /* * No need for a default case, since * ufs_qcom_testbus_cfg_is_ok() checks that the configuration * is legal */ } mask <<= offset; if (in_task()) spin_unlock_irqrestore(hba->host->host_lock, flags); ufshcd_rmwl(host->hba, TEST_BUS_SEL, (u32)host->testbus.select_major << 19, REG_UFS_CFG1); ufshcd_rmwl(host->hba, mask, (u32)host->testbus.select_minor << offset, reg); ufs_qcom_enable_test_bus(host); /* * Make sure the test bus configuration is * committed before returning. */ mb(); return 0; } static void ufs_qcom_testbus_read(struct ufs_hba *hba) { ufshcd_dump_regs(hba, UFS_TEST_BUS, 4, "UFS_TEST_BUS "); } static void ufs_qcom_print_unipro_testbus(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); u32 *testbus = NULL; int i, nminor = 256, testbus_len = nminor * sizeof(u32); testbus = kmalloc(testbus_len, GFP_KERNEL); if (!testbus) return; host->testbus.select_major = TSTBUS_UNIPRO; for (i = 0; i < nminor; i++) { host->testbus.select_minor = i; ufs_qcom_testbus_config(host); testbus[i] = ufshcd_readl(hba, UFS_TEST_BUS); } print_hex_dump(KERN_ERR, "UNIPRO_TEST_BUS ", DUMP_PREFIX_OFFSET, 16, 4, testbus, testbus_len, false); kfree(testbus); } static void ufs_qcom_print_utp_hci_testbus(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); u32 *testbus = NULL; int i, nminor = 32, testbus_len = nminor * sizeof(u32); testbus = kmalloc(testbus_len, GFP_KERNEL); if (!testbus) return; host->testbus.select_major = TSTBUS_UTP_HCI; for (i = 0; i < nminor; i++) { host->testbus.select_minor = i; ufs_qcom_testbus_config(host); testbus[i] = ufshcd_readl(hba, UFS_TEST_BUS); } print_hex_dump(KERN_ERR, "UTP_HCI_TEST_BUS ", DUMP_PREFIX_OFFSET, 16, 4, testbus, testbus_len, false); kfree(testbus); } static void ufs_qcom_dump_dbg_regs(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); struct phy *phy = host->generic_phy; host->err_occurred = true; ufshcd_dump_regs(hba, REG_UFS_SYS1CLK_1US, 16 * 4, "HCI Vendor Specific Registers "); ufshcd_dump_regs(hba, UFS_MEM_ICE, 29 * 4, "HCI Shared ICE Registers "); /* sleep a bit intermittently as we are dumping too much data */ ufs_qcom_print_hw_debug_reg_all(hba, NULL, ufs_qcom_dump_regs_wrapper); if (in_task()) { usleep_range(1000, 1100); ufs_qcom_testbus_read(hba); usleep_range(1000, 1100); ufs_qcom_print_unipro_testbus(hba); usleep_range(1000, 1100); ufs_qcom_print_utp_hci_testbus(hba); usleep_range(1000, 1100); ufs_qcom_phy_dbg_register_dump(phy); } BUG_ON(host->crash_on_err); } /* * ufs_qcom_parse_limits - read limits from DTS */ static void ufs_qcom_parse_limits(struct ufs_qcom_host *host) { struct device_node *np = host->hba->dev->of_node; u32 val; u32 dev_major = 0, dev_minor = 0; if (!np) return; host->limit_tx_hs_gear = host->max_hs_gear; host->limit_rx_hs_gear = host->max_hs_gear; host->limit_tx_pwm_gear = UFS_QCOM_LIMIT_PWMGEAR_TX; host->limit_rx_pwm_gear = UFS_QCOM_LIMIT_PWMGEAR_RX; host->limit_rate = UFS_QCOM_LIMIT_HS_RATE; host->limit_phy_submode = UFS_QCOM_LIMIT_PHY_SUBMODE; /* * The bootloader passes the on board device * information to the HLOS using the UFS host controller register's * UFS_MEM_DEBUG_SPARE_CFG Bit[0:3] = device's minor revision * UFS_MEM_DEBUG_SPARE_CFG Bit[4:7] = device's major revision * For example, UFS 3.1 devices would have a 0x31, and UFS 4.0 devices * would have a 0x40 as the content of the mentioned register. * If the bootloader does not support this feature, the default * hardcoded setting would be used. The DT settings can be used to * override any other gear's and Rate's settings. */ if (host->hw_ver.major >= 0x5) { val = ufshcd_readl(host->hba, REG_UFS_DEBUG_SPARE_CFG); dev_major = (val & UFS_DEVICE_VER_MAJOR_MASK) >> UFS_DEVICE_VER_MAJOR_SHFT; dev_minor = val & UFS_DEVICE_VER_MINOR_MASK; } if (host->hw_ver.major == 0x5 && dev_major == 0x4 && dev_minor == 0) { host->limit_rate = PA_HS_MODE_A; host->limit_phy_submode = UFS_QCOM_PHY_SUBMODE_G5; } of_property_read_u32(np, "limit-tx-hs-gear", &host->limit_tx_hs_gear); of_property_read_u32(np, "limit-rx-hs-gear", &host->limit_rx_hs_gear); of_property_read_u32(np, "limit-tx-pwm-gear", &host->limit_tx_pwm_gear); of_property_read_u32(np, "limit-rx-pwm-gear", &host->limit_rx_pwm_gear); of_property_read_u32(np, "limit-rate", &host->limit_rate); of_property_read_u32(np, "limit-phy-submode", &host->limit_phy_submode); if (of_property_read_bool(np, "limit-rate-ufs3")) ufs_qcom_set_rate_a(host); } /* * ufs_qcom_parse_g4_workaround_flag - read bypass-g4-cfgready entry from DT */ static void ufs_qcom_parse_g4_workaround_flag(struct ufs_qcom_host *host) { struct device_node *np = host->hba->dev->of_node; const char *str = "bypass-g4-cfgready"; if (!np) return; host->bypass_g4_cfgready = of_property_read_bool(np, str); } /* * ufs_qcom_parse_lpm - read from DTS whether LPM modes should be disabled. */ static void ufs_qcom_parse_lpm(struct ufs_qcom_host *host) { struct device_node *node = host->hba->dev->of_node; host->disable_lpm = of_property_read_bool(node, "qcom,disable-lpm"); if (host->disable_lpm) ufs_qcom_msg(INFO, host->hba->dev, "(%s) All LPM is disabled\n", __func__); of_property_read_u32(node, "qcom,vccq-lpm-uV", &host->vccq_lpm_uV); } /* * ufs_qcom_parse_wb - read from DTS whether WB support should be enabled. */ static void ufs_qcom_parse_wb(struct ufs_qcom_host *host) { struct device_node *node = host->hba->dev->of_node; host->disable_wb_support = of_property_read_bool(node, "qcom,disable-wb-support"); if (host->disable_wb_support) pr_info("%s: WB support disabled\n", __func__); } /** * ufs_qcom_device_reset() - toggle the (optional) device reset line * @hba: per-adapter instance * * Toggles the (optional) reset line to reset the attached device. */ static int ufs_qcom_device_reset(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int ret = 0; /* Reset UFS Host Controller and PHY */ ret = ufs_qcom_host_reset(hba); if (ret) ufs_qcom_msg(WARN, hba->dev, "%s: host reset returned %d\n", __func__, ret); #if defined(CONFIG_UFSFEATURE) ufsf_reset_host(ufs_qcom_get_ufsf(hba)); #endif /* reset gpio is optional */ if (!host->device_reset) return -EOPNOTSUPP; /* * The UFS device shall detect reset pulses of 1us, sleep for 10us to * be on the safe side. */ ufs_qcom_device_reset_ctrl(hba, true); usleep_range(10, 15); ufs_qcom_device_reset_ctrl(hba, false); usleep_range(10, 15); return 0; } #if IS_ENABLED(CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND) static void ufs_qcom_config_scaling_param(struct ufs_hba *hba, struct devfreq_dev_profile *p, void *data) { static struct devfreq_simple_ondemand_data *d; if (!data) return; d = (struct devfreq_simple_ondemand_data *)data; p->polling_ms = 60; p->timer = DEVFREQ_TIMER_DELAYED; d->upthreshold = 70; d->downdifferential = 65; } #else static void ufs_qcom_config_scaling_param(struct ufs_hba *hba, struct devfreq_dev_profile *p, void *data) { } #endif static struct ufs_dev_fix ufs_qcom_dev_fixups[] = { UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, UFS_DEVICE_QUIRK_PA_HIBER8TIME | UFS_DEVICE_QUIRK_PA_TX_HSG1_SYNC_LENGTH), UFS_FIX(UFS_VENDOR_MICRON, UFS_ANY_MODEL, UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM), UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL, UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM), UFS_FIX(UFS_VENDOR_WDC, UFS_ANY_MODEL, UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE), UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL, UFS_DEVICE_QUIRK_DELAY_AFTER_LPM), END_FIX }; static void ufs_qcom_fixup_dev_quirks(struct ufs_hba *hba) { ufshcd_fixup_dev_quirks(hba, ufs_qcom_dev_fixups); } /* * struct ufs_hba_qcom_vops - UFS QCOM specific variant operations * * The variant operations configure the necessary controller and PHY * handshake during initialization. */ static const struct ufs_hba_variant_ops ufs_hba_qcom_vops = { .name = "qcom", .init = ufs_qcom_init, .exit = ufs_qcom_exit, .get_ufs_hci_version = ufs_qcom_get_ufs_hci_version, .clk_scale_notify = ufs_qcom_clk_scale_notify, .event_notify = ufs_qcom_event_notify, .setup_clocks = ufs_qcom_setup_clocks, .hce_enable_notify = ufs_qcom_hce_enable_notify, .link_startup_notify = ufs_qcom_link_startup_notify, .pwr_change_notify = ufs_qcom_pwr_change_notify, .hibern8_notify = ufs_qcom_hibern8_notify, .apply_dev_quirks = ufs_qcom_apply_dev_quirks, .suspend = ufs_qcom_suspend, .resume = ufs_qcom_resume, .dbg_register_dump = ufs_qcom_dump_dbg_regs, .device_reset = ufs_qcom_device_reset, .config_scaling_param = ufs_qcom_config_scaling_param, .setup_xfer_req = ufs_qcom_qos, .program_key = ufs_qcom_ice_program_key, .fixup_dev_quirks = ufs_qcom_fixup_dev_quirks, }; /** * QCOM specific sysfs group and nodes */ static ssize_t err_state_show(struct device *dev, struct device_attribute *attr, char *buf) { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); return scnprintf(buf, PAGE_SIZE, "%d\n", !!host->err_occurred); } static DEVICE_ATTR_RO(err_state); static ssize_t power_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { struct ufs_hba *hba = dev_get_drvdata(dev); static const char * const names[] = { "INVALID MODE", "FAST MODE", "SLOW MODE", "INVALID MODE", "FASTAUTO MODE", "SLOWAUTO MODE", "INVALID MODE", }; /* Print current power info */ return scnprintf(buf, PAGE_SIZE, "[Rx,Tx]: Gear[%d,%d], Lane[%d,%d], PWR[%s,%s], Rate-%c\n", hba->pwr_info.gear_rx, hba->pwr_info.gear_tx, hba->pwr_info.lane_rx, hba->pwr_info.lane_tx, names[hba->pwr_info.pwr_rx], names[hba->pwr_info.pwr_tx], hba->pwr_info.hs_rate == PA_HS_MODE_B ? 'B' : 'A'); } static DEVICE_ATTR_RO(power_mode); static ssize_t bus_speed_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); return scnprintf(buf, PAGE_SIZE, "%d\n", !!atomic_read(&host->scale_up)); } static DEVICE_ATTR_RO(bus_speed_mode); static ssize_t clk_status_show(struct device *dev, struct device_attribute *attr, char *buf) { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); return scnprintf(buf, PAGE_SIZE, "%d\n", !!atomic_read(&host->clks_on)); } static DEVICE_ATTR_RO(clk_status); static unsigned int ufs_qcom_gec(struct ufs_hba *hba, u32 id, char *err_name) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); unsigned long flags; int i; struct ufs_event_hist *e; if (id >= UFS_EVT_CNT) return -EINVAL; e = &hba->ufs_stats.event[id]; spin_lock_irqsave(hba->host->host_lock, flags); for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) { int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH; if (e->tstamp[p] == 0) continue; ufs_qcom_msg(ERR, hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p, e->val[p], ktime_to_us(e->tstamp[p])); } spin_unlock_irqrestore(hba->host->host_lock, flags); return host->valid_evt_cnt[id]; } static ssize_t err_count_show(struct device *dev, struct device_attribute *attr, char *buf) { struct ufs_hba *hba = dev_get_drvdata(dev); return scnprintf(buf, PAGE_SIZE, "%s: %d\n%s: %d\n%s: %d\n%s: %d\n", "pa_err_cnt_total", ufs_qcom_gec(hba, UFS_EVT_PA_ERR, "pa_err_cnt_total"), "dl_err_cnt_total", ufs_qcom_gec(hba, UFS_EVT_DL_ERR, "dl_err_cnt_total"), "dme_err_cnt", ufs_qcom_gec(hba, UFS_EVT_DME_ERR, "dme_err_cnt"), "req_abort_cnt", hba->req_abort_count); } static DEVICE_ATTR_RO(err_count); static ssize_t dbg_state_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); bool v; if (!capable(CAP_SYS_ADMIN)) return -EACCES; if (kstrtobool(buf, &v)) return -EINVAL; host->dbg_en = v; return count; } static ssize_t dbg_state_show(struct device *dev, struct device_attribute *attr, char *buf) { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); return scnprintf(buf, PAGE_SIZE, "%d\n", host->dbg_en); } static DEVICE_ATTR_RW(dbg_state); static ssize_t crash_on_err_show(struct device *dev, struct device_attribute *attr, char *buf) { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); return scnprintf(buf, PAGE_SIZE, "%d\n", !!host->crash_on_err); } static ssize_t crash_on_err_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); bool v; if (!capable(CAP_SYS_ADMIN)) return -EACCES; if (kstrtobool(buf, &v)) return -EINVAL; host->crash_on_err = v; return count; } static DEVICE_ATTR_RW(crash_on_err); static ssize_t hibern8_count_show(struct device *dev, struct device_attribute *attr, char *buf) { u32 hw_h8_enter; u32 sw_h8_enter; u32 sw_hw_h8_enter; u32 hw_h8_exit; u32 sw_h8_exit; struct ufs_hba *hba = dev_get_drvdata(dev); pm_runtime_get_sync(hba->dev); ufshcd_hold(hba, false); hw_h8_enter = ufshcd_readl(hba, REG_UFS_HW_H8_ENTER_CNT); sw_h8_enter = ufshcd_readl(hba, REG_UFS_SW_H8_ENTER_CNT); sw_hw_h8_enter = ufshcd_readl(hba, REG_UFS_SW_AFTER_HW_H8_ENTER_CNT); hw_h8_exit = ufshcd_readl(hba, REG_UFS_HW_H8_EXIT_CNT); sw_h8_exit = ufshcd_readl(hba, REG_UFS_SW_H8_EXIT_CNT); ufshcd_release(hba); pm_runtime_put_sync(hba->dev); return scnprintf(buf, PAGE_SIZE, "%s: %d\n%s: %d\n%s: %d\n%s: %d\n%s: %d\n", "hw_h8_enter", hw_h8_enter, "sw_h8_enter", sw_h8_enter, "sw_after_hw_h8_enter", sw_hw_h8_enter, "hw_h8_exit", hw_h8_exit, "sw_h8_exit", sw_h8_exit); } static DEVICE_ATTR_RO(hibern8_count); static ssize_t ber_th_exceeded_show(struct device *dev, struct device_attribute *attr, char *buf) { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); return scnprintf(buf, PAGE_SIZE, "%d\n", host->ber_th_exceeded); } static DEVICE_ATTR_RO(ber_th_exceeded); static ssize_t irq_affinity_support_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); bool value; if (!capable(CAP_SYS_ADMIN)) return -EACCES; if (kstrtobool(buf, &value)) return -EINVAL; /* if current irq_affinity_support is same as requested one, don't proceed */ if (value == host->irq_affinity_support) goto out; /* if perf-mask is not set, don't proceed */ if (!host->perf_mask.bits[0]) goto out; host->irq_affinity_support = !!value; /* Reset cpu affinity accordingly */ if (host->irq_affinity_support) ufs_qcom_set_affinity_hint(hba, true); else ufs_qcom_set_affinity_hint(hba, false); return count; out: return -EINVAL; } static ssize_t irq_affinity_support_show(struct device *dev, struct device_attribute *attr, char *buf) { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); return scnprintf(buf, PAGE_SIZE, "%d\n", !!host->irq_affinity_support); } static DEVICE_ATTR_RW(irq_affinity_support); static ssize_t ufs_pm_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { int ret = -1; struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); switch (host->ufs_pm_mode) { case 0: ret = scnprintf(buf, 6, "NONE\n"); break; case 1: ret = scnprintf(buf, 5, "S2R\n"); break; case 2: ret = scnprintf(buf, 12, "DEEPSLEEP\n"); break; default: break; } return ret; } static ssize_t ufs_pm_mode_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { char kbuff[12] = {0}; struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_qcom_host *host = ufshcd_get_variant(hba); if (!buf) return -EINVAL; strscpy(kbuff, buf, 11); if (!strncasecmp(kbuff, "NONE", 4)) host->ufs_pm_mode = 0; else if (!strncasecmp(kbuff, "S2R", 3)) host->ufs_pm_mode = 1; else if (!strncasecmp(kbuff, "DEEPSLEEP", 9)) host->ufs_pm_mode = 2; else dev_err(hba->dev, "Invalid entry for ufs_pm_mode\n"); return count; } static DEVICE_ATTR_RW(ufs_pm_mode); static struct attribute *ufs_qcom_sysfs_attrs[] = { &dev_attr_err_state.attr, &dev_attr_power_mode.attr, &dev_attr_bus_speed_mode.attr, &dev_attr_clk_status.attr, &dev_attr_err_count.attr, &dev_attr_dbg_state.attr, &dev_attr_crash_on_err.attr, &dev_attr_hibern8_count.attr, &dev_attr_ber_th_exceeded.attr, &dev_attr_irq_affinity_support.attr, &dev_attr_ufs_pm_mode.attr, NULL }; static const struct attribute_group ufs_qcom_sysfs_group = { .name = "qcom", .attrs = ufs_qcom_sysfs_attrs, }; static int ufs_qcom_init_sysfs(struct ufs_hba *hba) { int ret; ret = sysfs_create_group(&hba->dev->kobj, &ufs_qcom_sysfs_group); if (ret) ufs_qcom_msg(ERR, hba->dev, "%s: Failed to create qcom sysfs group (err = %d)\n", __func__, ret); return ret; } static void ufs_qcom_hook_send_command(void *param, struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); #if defined(CONFIG_UFSFEATURE) ufs_vh_send_command(param, hba, lrbp); #endif if (lrbp && lrbp->cmd && lrbp->cmd->cmnd[0]) { struct request *rq = scsi_cmd_to_rq(lrbp->cmd); int sz = rq ? blk_rq_sectors(rq) : 0; ufs_qcom_log_str(host, "<,%x,%d,%x,%d\n", lrbp->cmd->cmnd[0], lrbp->task_tag, ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL), sz); if (host->dbg_en) trace_ufs_qcom_command(dev_name(hba->dev), UFS_QCOM_CMD_SEND, lrbp->cmd->cmnd[0], lrbp->task_tag, ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL), sz); if ((host->irq_affinity_support) && atomic_read(&host->hi_pri_en) && rq) rq->cmd_flags |= REQ_HIPRI; } } static void ufs_qcom_hook_compl_command(void *param, struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); #if defined(CONFIG_UFSFEATURE) ufs_vh_compl_command(param, hba, lrbp); #endif if (lrbp && lrbp->cmd) { int sz = scsi_cmd_to_rq(lrbp->cmd) ? blk_rq_sectors(scsi_cmd_to_rq(lrbp->cmd)) : 0; ufs_qcom_log_str(host, ">,%x,%d,%x,%d\n", lrbp->cmd->cmnd[0], lrbp->task_tag, ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL), sz); if (host->dbg_en) trace_ufs_qcom_command(dev_name(hba->dev), UFS_QCOM_CMD_COMPL, lrbp->cmd->cmnd[0], lrbp->task_tag, ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL), sz); } } static void ufs_qcom_hook_send_uic_command(void *param, struct ufs_hba *hba, struct uic_command *ucmd, int str_t) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); unsigned int a, b, c, cmd; char ch; if (str_t == UFS_CMD_SEND) { a = ucmd->argument1; b = ucmd->argument2; c = ucmd->argument3; cmd = ucmd->command; ch = '('; if (host->dbg_en) trace_ufs_qcom_uic(dev_name(hba->dev), UFS_QCOM_CMD_SEND, cmd, a, b, c); } else { a = ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1), b = ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2), c = ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3); cmd = ufshcd_readl(hba, REG_UIC_COMMAND); ch = ')'; if (host->dbg_en) trace_ufs_qcom_uic(dev_name(hba->dev), UFS_QCOM_CMD_COMPL, cmd, a, b, c); } ufs_qcom_log_str(host, "%c,%x,%x,%x,%x\n", ch, cmd, a, b, c); } static void ufs_qcom_hook_send_tm_command(void *param, struct ufs_hba *hba, int tag, int str) { } static void ufs_qcom_hook_check_int_errors(void *param, struct ufs_hba *hba, bool queue_eh_work) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); if (queue_eh_work) { ufs_qcom_log_str(host, "_,%x,%x\n", hba->errors, hba->uic_error); if (host->dbg_en) trace_ufs_qcom_hook_check_int_errors(dev_name(hba->dev), hba->errors, hba->uic_error); } } static void ufs_qcom_update_sdev(void *param, struct scsi_device *sdev) { #if defined(CONFIG_UFSFEATURE) ufs_vh_update_sdev(param, sdev); #endif sdev->broken_fua = 1; } static void ufs_qcom_hook_prepare_command(void *param, struct ufs_hba *hba, struct request *rq, struct ufshcd_lrb *lrbp, int *err) { #if IS_ENABLED(CONFIG_QTI_CRYPTO_FDE) struct ice_data_setting setting; if (!rq->crypt_keyslot) { if (!crypto_qti_ice_config_start(rq, &setting)) { if ((rq_data_dir(rq) == WRITE) ? setting.encr_bypass : setting.decr_bypass) { lrbp->crypto_key_slot = -1; } else { lrbp->crypto_key_slot = setting.crypto_data.key_index; lrbp->data_unit_num = rq->bio->bi_iter.bi_sector >> ICE_CRYPTO_DATA_UNIT_4_KB; } } } else { lrbp->crypto_key_slot = blk_ksm_get_slot_idx(rq->crypt_keyslot); lrbp->data_unit_num = rq->crypt_ctx->bc_dun[0]; } #endif } /* * Refer: common/include/trace/hooks/ufshcd.h for available hooks */ static void ufs_qcom_register_hooks(void) { #if defined(CONFIG_UFSFEATURE) register_trace_android_vh_ufs_prepare_command(ufs_vh_prep_fn, NULL); #endif register_trace_android_vh_ufs_send_command(ufs_qcom_hook_send_command, NULL); register_trace_android_vh_ufs_compl_command( ufs_qcom_hook_compl_command, NULL); register_trace_android_vh_ufs_send_uic_command( ufs_qcom_hook_send_uic_command, NULL); register_trace_android_vh_ufs_send_tm_command( ufs_qcom_hook_send_tm_command, NULL); register_trace_android_vh_ufs_check_int_errors( ufs_qcom_hook_check_int_errors, NULL); register_trace_android_vh_ufs_update_sdev(ufs_qcom_update_sdev, NULL); register_trace_android_vh_ufs_prepare_command( ufs_qcom_hook_prepare_command, NULL); } #ifdef CONFIG_ARM_QCOM_CPUFREQ_HW static int ufs_cpufreq_status(void) { struct cpufreq_policy *policy; policy = cpufreq_cpu_get(0); if (!policy) { ufs_qcom_msg(WARN, NULL, "cpufreq not probed yet, defer once\n"); return -EPROBE_DEFER; } cpufreq_cpu_put(policy); return 0; } #else static int ufs_cpufreq_status(void) { return 0; } #endif static u32 is_bootdevice_ufs = UFS_BOOT_DEVICE; static int ufs_qcom_read_boot_config(struct platform_device *pdev) { u8 *buf; size_t len; struct nvmem_cell *cell; cell = nvmem_cell_get(&pdev->dev, "boot_conf"); if (IS_ERR(cell)) return -EINVAL; buf = (u8 *)nvmem_cell_read(cell, &len); if (IS_ERR(buf)) return -EINVAL; is_bootdevice_ufs = (*buf) >> 1 & 0x1f; dev_err(&pdev->dev, "boot_config val = %x is_bootdevice_ufs = %x\n", *buf, is_bootdevice_ufs); kfree(buf); nvmem_cell_put(cell); return is_bootdevice_ufs; } /** * ufs_qcom_probe - probe routine of the driver * @pdev: pointer to Platform device handle * * Return zero for success and non-zero for failure */ static int ufs_qcom_probe(struct platform_device *pdev) { int err = 0; struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; if (!ufs_qcom_read_boot_config(pdev)) { dev_err(dev, "UFS is not boot dev.\n"); return err; } /** * CPUFreq driver is needed for performance reasons. * Assumption - cpufreq gets probed the second time. * If cpufreq is not probed by the time the driver requests for cpu * policy later on, cpufreq would be disabled and performance would be * impacted adversly. */ err = ufs_cpufreq_status(); if (err) return err; /* * On qcom platforms, bootdevice is the primary storage * device. This device can either be eMMC or UFS. * The type of device connected is detected at runtime. * So, if an eMMC device is connected, and this function * is invoked, it would turn-off the regulator if it detects * that the storage device is not ufs. * These regulators are turned ON by the bootloaders & turning * them off without sending PON may damage the connected device. * Hence, check for the connected device early-on & don't turn-off * the regulators. */ /* *Defer secondary UFS device probe if all the LUNS of *primary UFS boot device are not enumerated. */ if ((of_property_read_bool(np, "secondary-storage")) && (!ufs_qcom_hosts[0] || !(ufs_qcom_hosts[0]->hba->luns_avail == 1))) { err = -EPROBE_DEFER; return err; } /* Perform generic probe */ err = ufshcd_pltfrm_init(pdev, &ufs_hba_qcom_vops); if (err) ufs_qcom_msg(ERR, dev, "ufshcd_pltfrm_init() failed %d\n", err); if (!(of_property_read_bool(np, "secondary-storage"))) ufs_qcom_register_hooks(); return err; } /** * ufs_qcom_remove - set driver_data of the device to NULL * @pdev: pointer to platform device handle * * Always returns 0 */ static int ufs_qcom_remove(struct platform_device *pdev) { struct ufs_hba *hba; struct ufs_qcom_host *host; struct ufs_qcom_qos_req *r; struct qos_cpu_group *qcg; int i; if (!is_bootdevice_ufs) { dev_info(&pdev->dev, "UFS is not boot dev.\n"); return 0; } hba = platform_get_drvdata(pdev); host = ufshcd_get_variant(hba); r = host->ufs_qos; qcg = r->qcg; if (msm_minidump_enabled()) atomic_notifier_chain_unregister(&panic_notifier_list, &host->ufs_qcom_panic_nb); pm_runtime_get_sync(&(pdev)->dev); for (i = 0; i < r->num_groups; i++, qcg++) remove_group_qos(qcg); #if defined(CONFIG_UFSFEATURE) ufsf_remove(ufs_qcom_get_ufsf(hba)); #endif ufshcd_remove(hba); return 0; } /** * ufs_qcom_enable_vccq_shutdown - read from DTS whether vccq_shutdown * should be enabled for puting additional vote on VCCQ LDO during shutdown. * @hba: per adapter instance */ static void ufs_qcom_enable_vccq_shutdown(struct ufs_hba *hba) { struct ufs_qcom_host *host = ufshcd_get_variant(hba); int err; err = ufs_qcom_parse_reg_info(host, "qcom,vccq-shutdown", &host->vccq_shutdown); if (host->vccq_shutdown) { err = ufs_qcom_enable_vreg(hba->dev, host->vccq_shutdown); if (err) ufs_qcom_msg(ERR, hba->dev, "%s: failed enable vccq_shutdown err=%d\n", __func__, err); } } static void ufs_qcom_shutdown(struct platform_device *pdev) { struct ufs_hba *hba; struct ufs_qcom_host *host; if (!is_bootdevice_ufs) { dev_info(&pdev->dev, "UFS is not boot dev.\n"); return; } hba = platform_get_drvdata(pdev); host = ufshcd_get_variant(hba); ufs_qcom_log_str(host, "0xdead\n"); if (host->dbg_en) trace_ufs_qcom_shutdown(dev_name(hba->dev)); /* put an additional vote on UFS VCCQ LDO if required */ ufs_qcom_enable_vccq_shutdown(hba); ufshcd_pltfrm_shutdown(pdev); /* UFS_RESET TLMM register cannot reset to POR value '1' after warm * reset, so deassert ufs device reset line after UFS device shutdown * to ensure the UFS_RESET TLMM register value is POR value */ if (!host->bypass_pbl_rst_wa) ufs_qcom_device_reset_ctrl(hba, false); } static int ufs_qcom_system_suspend(struct device *dev) { struct device_node *np = dev->of_node; if (of_property_read_bool(np, "non-removable") && !is_bootdevice_ufs) { dev_info(dev, "UFS is not boot dev.\n"); return 0; } return ufshcd_system_suspend(dev); } static int ufs_qcom_system_resume(struct device *dev) { if (!is_bootdevice_ufs) { dev_info(dev, "UFS is not boot dev.\n"); return 0; } return ufshcd_system_resume(dev); } static int ufs_qcom_suspend_prepare(struct device *dev) { struct ufs_hba *hba; struct ufs_qcom_host *host; if (!is_bootdevice_ufs) { dev_info(dev, "UFS is not boot dev.\n"); return 0; } hba = dev_get_drvdata(dev); host = ufshcd_get_variant(hba); host->spm_lvl_prev = hba->spm_lvl; /* * For deep sleep, if "set_ds_spm_level" flag is true, set the * spm level to lvl 5 because all regulators is turned off in DS. * For other scenarios like s2idle, retain the default spm level. */ switch (host->ufs_pm_mode) { case UFS_QCOM_SYSFS_NONE: if (host->set_ds_spm_level && (pm_suspend_target_state == PM_SUSPEND_MEM)) hba->spm_lvl = UFS_PM_LVL_5; break; case UFS_QCOM_SYSFS_DEEPSLEEP: if (host->set_ds_spm_level) hba->spm_lvl = UFS_PM_LVL_5; break; case UFS_QCOM_SYSFS_S2R: default: break; } if (hba->spm_lvl != host->spm_lvl_prev) dev_info(dev, "spm level is changed from %d to %d\n", host->spm_lvl_prev, hba->spm_lvl); return ufshcd_suspend_prepare(dev); } static void ufs_qcom_resume_complete(struct device *dev) { struct ufs_hba *hba; struct ufs_qcom_host *host; if (!is_bootdevice_ufs) { dev_info(dev, "UFS is not boot dev.\n"); return; } hba = dev_get_drvdata(dev); host = ufshcd_get_variant(hba); if (host->set_ds_spm_level) hba->spm_lvl = host->spm_lvl_prev; host->ufs_pm_mode = UFS_QCOM_SYSFS_NONE; return ufshcd_resume_complete(dev); } static const struct of_device_id ufs_qcom_of_match[] = { { .compatible = "qcom,ufshc"}, {}, }; MODULE_DEVICE_TABLE(of, ufs_qcom_of_match); #ifdef CONFIG_ACPI static const struct acpi_device_id ufs_qcom_acpi_match[] = { { "QCOM24A5" }, { }, }; MODULE_DEVICE_TABLE(acpi, ufs_qcom_acpi_match); #endif static const struct dev_pm_ops ufs_qcom_pm_ops = { SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL) .prepare = ufs_qcom_suspend_prepare, .complete = ufs_qcom_resume_complete, #ifdef CONFIG_PM_SLEEP .suspend = ufs_qcom_system_suspend, .resume = ufs_qcom_system_resume, .freeze = ufshcd_system_freeze, .restore = ufshcd_system_restore, .thaw = ufshcd_system_thaw, #endif }; static struct platform_driver ufs_qcom_pltform = { .probe = ufs_qcom_probe, .remove = ufs_qcom_remove, .shutdown = ufs_qcom_shutdown, .driver = { .name = "ufshcd-qcom", .pm = &ufs_qcom_pm_ops, .of_match_table = of_match_ptr(ufs_qcom_of_match), .acpi_match_table = ACPI_PTR(ufs_qcom_acpi_match), }, }; module_platform_driver(ufs_qcom_pltform); MODULE_LICENSE("GPL v2");