/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2021 - Google LLC * Author: David Brazdil */ #ifndef __ARM64_KVM_S2MPU_H__ #define __ARM64_KVM_S2MPU_H__ #include #include #define S2MPU_MMIO_SIZE SZ_64K #define SYSMMU_SYNC_MMIO_SIZE SZ_64K #define SYSMMU_SYNC_S2_OFFSET SZ_32K #define SYSMMU_SYNC_S2_MMIO_SIZE (SYSMMU_SYNC_MMIO_SIZE - \ SYSMMU_SYNC_S2_OFFSET) #define NR_VIDS 8 #define NR_CTX_IDS 8 #define ALL_VIDS_BITMAP GENMASK(NR_VIDS - 1, 0) #define REG_NS_CTRL0 0x0 #define REG_NS_CTRL1 0x4 #define REG_NS_CFG 0x10 #define REG_NS_INTERRUPT_ENABLE_PER_VID_SET 0x20 #define REG_NS_INTERRUPT_CLEAR 0x2c #define REG_NS_VERSION 0x60 #define REG_NS_INFO 0x64 #define REG_NS_STATUS 0x68 #define REG_NS_NUM_CONTEXT 0x100 #define REG_NS_CONTEXT_CFG_VALID_VID 0x104 #define REG_NS_ALL_INVALIDATION 0x1000 #define REG_NS_RANGE_INVALIDATION 0x1020 #define REG_NS_RANGE_INVALIDATION_START_PPN 0x1024 #define REG_NS_RANGE_INVALIDATION_END_PPN 0x1028 #define REG_NS_FAULT_STATUS 0x2000 #define REG_NS_FAULT_PA_LOW(vid) (0x2004 + ((vid) * 0x20)) #define REG_NS_FAULT_PA_HIGH(vid) (0x2008 + ((vid) * 0x20)) #define REG_NS_FAULT_INFO(vid) (0x2010 + ((vid) * 0x20)) #define REG_NS_READ_MPTC 0x3000 #define REG_NS_READ_MPTC_TAG_PPN 0x3004 #define REG_NS_READ_MPTC_TAG_OTHERS 0x3008 #define REG_NS_READ_MPTC_DATA 0x3010 #define REG_NS_L1ENTRY_L2TABLE_ADDR(vid, gb) (0x4000 + ((vid) * 0x200) + ((gb) * 0x8)) #define REG_NS_L1ENTRY_ATTR(vid, gb) (0x4004 + ((vid) * 0x200) + ((gb) * 0x8)) #define CTRL0_ENABLE BIT(0) #define CTRL0_INTERRUPT_ENABLE BIT(1) #define CTRL0_FAULT_RESP_TYPE_SLVERR BIT(2) /* for v8 */ #define CTRL0_FAULT_RESP_TYPE_DECERR BIT(2) /* for v9 */ #define CTRL0_MASK (CTRL0_ENABLE | \ CTRL0_INTERRUPT_ENABLE | \ CTRL0_FAULT_RESP_TYPE_SLVERR | \ CTRL0_FAULT_RESP_TYPE_DECERR) #define CTRL1_DISABLE_CHK_S1L1PTW BIT(0) #define CTRL1_DISABLE_CHK_S1L2PTW BIT(1) #define CTRL1_ENABLE_PAGE_SIZE_AWARENESS BIT(2) #define CTRL1_DISABLE_CHK_USER_MATCHED_REQ BIT(3) #define CTRL1_MASK (CTRL1_DISABLE_CHK_S1L1PTW | \ CTRL1_DISABLE_CHK_S1L2PTW | \ CTRL1_ENABLE_PAGE_SIZE_AWARENESS | \ CTRL1_DISABLE_CHK_USER_MATCHED_REQ) #define CFG_MPTW_CACHE_OVERRIDE BIT(0) #define CFG_MPTW_CACHE_VALUE GENMASK(7, 4) #define CFG_MPTW_QOS_OVERRIDE BIT(8) #define CFG_MPTW_QOS_VALUE GENMASK(15, 12) #define CFG_MPTW_SHAREABLE BIT(16) #define CFG_MASK (CFG_MPTW_CACHE_OVERRIDE | \ CFG_MPTW_CACHE_VALUE | \ CFG_MPTW_QOS_OVERRIDE | \ CFG_MPTW_QOS_VALUE | \ CFG_MPTW_SHAREABLE) /* For use with hi_lo_readq_relaxed(). */ #define REG_NS_FAULT_PA_HIGH_LOW(vid) REG_NS_FAULT_PA_LOW(vid) /* Mask used for extracting VID from FAULT_* register offset. */ #define REG_NS_FAULT_VID_MASK GENMASK(7, 5) #define VERSION_MAJOR_ARCH_VER_MASK GENMASK(31, 28) #define VERSION_MINOR_ARCH_VER_MASK GENMASK(27, 24) #define VERSION_REV_ARCH_VER_MASK GENMASK(23, 16) #define VERSION_RTL_VER_MASK GENMASK(7, 0) /* Ignore RTL version in driver version check. */ #define VERSION_CHECK_MASK (VERSION_MAJOR_ARCH_VER_MASK | \ VERSION_MINOR_ARCH_VER_MASK | \ VERSION_REV_ARCH_VER_MASK) #define INFO_NUM_SET_MASK GENMASK(15, 0) #define STATUS_BUSY BIT(0) #define STATUS_ON_INVALIDATING BIT(1) #define NUM_CONTEXT_MASK GENMASK(3, 0) #define CONTEXT_CFG_VALID_VID_CTX_VALID(ctx) BIT((4 * (ctx)) + 3) #define CONTEXT_CFG_VALID_VID_CTX_VID(ctx, vid) \ FIELD_PREP(GENMASK((4 * (ctx) + 2), 4 * (ctx)), (vid)) #define INVALIDATION_INVALIDATE BIT(0) #define RANGE_INVALIDATION_PPN_SHIFT 12 #define NR_FAULT_INFO_REGS 8 #define FAULT_INFO_VID_MASK GENMASK(26, 24) #define FAULT_INFO_TYPE_MASK GENMASK(23, 21) #define FAULT_INFO_TYPE_CONTEXT 0x4 /* v9 only */ #define FAULT_INFO_TYPE_AP 0x2 #define FAULT_INFO_TYPE_MPTW 0x1 #define FAULT_INFO_RW_BIT BIT(20) #define FAULT_INFO_LEN_MASK GENMASK(19, 16) #define FAULT_INFO_ID_MASK GENMASK(15, 0) #define L1ENTRY_L2TABLE_ADDR_SHIFT 4 #define L1ENTRY_L2TABLE_ADDR(pa) ((pa) >> L1ENTRY_L2TABLE_ADDR_SHIFT) #define READ_MPTC_WAY_MASK GENMASK(18, 16) #define READ_MPTC_SET_MASK GENMASK(15, 0) #define READ_MPTC_MASK (READ_MPTC_WAY_MASK | READ_MPTC_SET_MASK) #define READ_MPTC_WAY(way) FIELD_PREP(READ_MPTC_WAY_MASK, (way)) #define READ_MPTC_SET(set) FIELD_PREP(READ_MPTC_SET_MASK, (set)) #define READ_MPTC(set, way) (READ_MPTC_SET(set) | READ_MPTC_WAY(way)) #define READ_MPTC_TAG_PPN_MASK GENMASK(23, 0) #define READ_MPTC_TAG_OTHERS_VID_MASK GENMASK(10, 8) #define READ_MPTC_TAG_OTHERS_GRAN_MASK GENMASK(5, 4) #define READ_MPTC_TAG_OTHERS_VALID_BIT BIT(0) #define READ_MPTC_TAG_OTHERS_MASK (READ_MPTC_TAG_OTHERS_VID_MASK | \ READ_MPTC_TAG_OTHERS_GRAN_MASK | \ READ_MPTC_TAG_OTHERS_VALID_BIT) #define L1ENTRY_ATTR_L2TABLE_EN BIT(0) #define L1ENTRY_ATTR_GRAN_4K 0x0 #define L1ENTRY_ATTR_GRAN_64K 0x1 #define L1ENTRY_ATTR_GRAN_2M 0x2 #define L1ENTRY_ATTR_PROT_MASK GENMASK(2, 1) #define L1ENTRY_ATTR_GRAN_MASK GENMASK(5, 4) #define L1ENTRY_ATTR_PROT(prot) FIELD_PREP(L1ENTRY_ATTR_PROT_MASK, prot) #define L1ENTRY_ATTR_GRAN(gran) FIELD_PREP(L1ENTRY_ATTR_GRAN_MASK, gran) #define L1ENTRY_ATTR_1G(prot) L1ENTRY_ATTR_PROT(prot) #define L1ENTRY_ATTR_L2(gran) (L1ENTRY_ATTR_GRAN(gran) | \ L1ENTRY_ATTR_L2TABLE_EN) #define NR_GIGABYTES 64 #define RO_GIGABYTES_FIRST 4 #define RO_GIGABYTES_LAST 33 #define NR_RO_GIGABYTES (RO_GIGABYTES_LAST - RO_GIGABYTES_FIRST + 1) #define NR_RW_GIGABYTES (NR_GIGABYTES - NR_RO_GIGABYTES) #ifdef CONFIG_ARM64_64K_PAGES #define SMPT_GRAN SZ_64K #define SMPT_GRAN_ATTR L1ENTRY_ATTR_GRAN_64K #else #define SMPT_GRAN SZ_4K #define SMPT_GRAN_ATTR L1ENTRY_ATTR_GRAN_4K #endif static_assert(SMPT_GRAN <= PAGE_SIZE); #define MPT_PROT_BITS 2 #define SMPT_WORD_SIZE sizeof(u32) #define SMPT_ELEMS_PER_BYTE (BITS_PER_BYTE / MPT_PROT_BITS) #define SMPT_ELEMS_PER_WORD (SMPT_WORD_SIZE * SMPT_ELEMS_PER_BYTE) #define SMPT_WORD_BYTE_RANGE (SMPT_GRAN * SMPT_ELEMS_PER_WORD) #define SMPT_NUM_ELEMS (SZ_1G / SMPT_GRAN) #define SMPT_SIZE (SMPT_NUM_ELEMS / SMPT_ELEMS_PER_BYTE) #define SMPT_NUM_WORDS (SMPT_SIZE / SMPT_WORD_SIZE) #define SMPT_NUM_PAGES (SMPT_SIZE / PAGE_SIZE) #define SMPT_ORDER get_order(SMPT_SIZE) /* SysMMU_SYNC registers, relative to SYSMMU_SYNC_S2_OFFSET. */ #define REG_NS_SYNC_CMD 0x0 #define REG_NS_SYNC_COMP 0x4 #define SYNC_CMD_SYNC BIT(0) #define SYNC_COMP_COMPLETE BIT(0) /* * Iterate over S2MPU gigabyte regions. Skip those that cannot be modified * (the MMIO registers are read only, with reset value MPT_PROT_NONE). */ #define for_each_gb_in_range(i, first, last) \ for ((i) = (first); (i) <= (last) && (i) < NR_GIGABYTES; \ (i) = (((i) + 1 == RO_GIGABYTES_FIRST) ? RO_GIGABYTES_LAST : (i)) + 1) #define for_each_gb(i) for_each_gb_in_range(i, 0, NR_GIGABYTES - 1) #define for_each_vid(i) for ((i) = 0; (i) < NR_VIDS; (i)++) #define for_each_gb_and_vid(gb, vid) for_each_vid((vid)) for_each_gb((gb)) enum s2mpu_version { S2MPU_VERSION_8 = 0x11000000, S2MPU_VERSION_9 = 0x20000000, }; enum mpt_prot { MPT_PROT_NONE = 0, MPT_PROT_R = BIT(0), MPT_PROT_W = BIT(1), MPT_PROT_RW = MPT_PROT_R | MPT_PROT_W, MPT_PROT_MASK = MPT_PROT_RW, }; static const u64 mpt_prot_doubleword[] = { [MPT_PROT_NONE] = 0x0000000000000000, [MPT_PROT_R] = 0x5555555555555555, [MPT_PROT_W] = 0xaaaaaaaaaaaaaaaa, [MPT_PROT_RW] = 0xffffffffffffffff, }; enum mpt_update_flags { MPT_UPDATE_L1 = BIT(0), MPT_UPDATE_L2 = BIT(1), }; struct fmpt { u32 *smpt; bool gran_1g; enum mpt_prot prot; enum mpt_update_flags flags; }; struct mpt { struct fmpt fmpt[NR_GIGABYTES]; }; /* Set protection bits of SMPT in a given range without using memset. */ static inline void __set_smpt_range_slow(u32 *smpt, size_t start_gb_byte, size_t end_gb_byte, enum mpt_prot prot) { size_t i, start_word_byte, end_word_byte, word_idx, first_elem, last_elem; u32 val; /* Iterate over u32 words. */ start_word_byte = start_gb_byte; while (start_word_byte < end_gb_byte) { /* Determine the range of bytes covered by this word. */ word_idx = start_word_byte / SMPT_WORD_BYTE_RANGE; end_word_byte = min( ALIGN(start_word_byte + 1, SMPT_WORD_BYTE_RANGE), end_gb_byte); /* Identify protection bit offsets within the word. */ first_elem = (start_word_byte / SMPT_GRAN) % SMPT_ELEMS_PER_WORD; last_elem = ((end_word_byte - 1) / SMPT_GRAN) % SMPT_ELEMS_PER_WORD; /* Modify the corresponding word. */ val = READ_ONCE(smpt[word_idx]); for (i = first_elem; i <= last_elem; i++) { val &= ~(MPT_PROT_MASK << (i * MPT_PROT_BITS)); val |= prot << (i * MPT_PROT_BITS); } WRITE_ONCE(smpt[word_idx], val); start_word_byte = end_word_byte; } } /* Set protection bits of SMPT in a given range. */ static inline void __set_smpt_range(u32 *smpt, size_t start_gb_byte, size_t end_gb_byte, enum mpt_prot prot) { size_t interlude_start, interlude_end, interlude_bytes, word_idx; char prot_byte = (char)mpt_prot_doubleword[prot]; if (start_gb_byte >= end_gb_byte) return; /* Check if range spans at least one full u32 word. */ interlude_start = ALIGN(start_gb_byte, SMPT_WORD_BYTE_RANGE); interlude_end = ALIGN_DOWN(end_gb_byte, SMPT_WORD_BYTE_RANGE); /* If not, fall back to editing bits in the given range. */ if (interlude_start >= interlude_end) { __set_smpt_range_slow(smpt, start_gb_byte, end_gb_byte, prot); return; } /* Use bit-editing for prologue/epilogue, memset for interlude. */ word_idx = interlude_start / SMPT_WORD_BYTE_RANGE; interlude_bytes = (interlude_end - interlude_start) / SMPT_GRAN / SMPT_ELEMS_PER_BYTE; __set_smpt_range_slow(smpt, start_gb_byte, interlude_start, prot); memset(&smpt[word_idx], prot_byte, interlude_bytes); __set_smpt_range_slow(smpt, interlude_end, end_gb_byte, prot); } /* Returns true if all SMPT protection bits match 'prot'. */ static inline bool __is_smpt_uniform(u32 *smpt, enum mpt_prot prot) { size_t i; u64 *doublewords = (u64 *)smpt; for (i = 0; i < SMPT_NUM_WORDS / 2; i++) { if (doublewords[i] != mpt_prot_doubleword[prot]) return false; } return true; } /* * Set protection bits of FMPT/SMPT in a given range. * Returns flags specifying whether L1/L2 changes need to be made visible * to the device. */ static inline void __set_fmpt_range(struct fmpt *fmpt, size_t start_gb_byte, size_t end_gb_byte, enum mpt_prot prot) { if (start_gb_byte == 0 && end_gb_byte >= SZ_1G) { /* Update covers the entire GB region. */ if (fmpt->gran_1g && fmpt->prot == prot) { fmpt->flags = 0; return; } fmpt->gran_1g = true; fmpt->prot = prot; fmpt->flags = MPT_UPDATE_L1; return; } if (fmpt->gran_1g) { /* GB region currently uses 1G mapping. */ if (fmpt->prot == prot) { fmpt->flags = 0; return; } /* * Range has different mapping than the rest of the GB. * Convert to PAGE_SIZE mapping. */ fmpt->gran_1g = false; __set_smpt_range(fmpt->smpt, 0, start_gb_byte, fmpt->prot); __set_smpt_range(fmpt->smpt, start_gb_byte, end_gb_byte, prot); __set_smpt_range(fmpt->smpt, end_gb_byte, SZ_1G, fmpt->prot); fmpt->flags = MPT_UPDATE_L1 | MPT_UPDATE_L2; return; } /* GB region currently uses PAGE_SIZE mapping. */ __set_smpt_range(fmpt->smpt, start_gb_byte, end_gb_byte, prot); /* Check if the entire GB region has the same prot bits. */ if (!__is_smpt_uniform(fmpt->smpt, prot)) { fmpt->flags = MPT_UPDATE_L2; return; } fmpt->gran_1g = true; fmpt->prot = prot; fmpt->flags = MPT_UPDATE_L1; } #endif /* __ARM64_KVM_S2MPU_H__ */