@@ -86,6 +86,7 @@ config ERRATA_THEAD_MAE
bool "Apply T-Head's memory attribute extension (XTheadMae) errata"
depends on ERRATA_THEAD && 64BIT && MMU
select RISCV_ALTERNATIVE_EARLY
+ select RISCV_ISA_SVPBMT
default y
help
This will apply the memory attribute extension errata to handle the
@@ -58,51 +58,6 @@ asm(ALTERNATIVE("sfence.vma %0, %1", "sfence.vma", SIFIVE_VENDOR_ID, \
ERRATA_SIFIVE_CIP_1200, CONFIG_ERRATA_SIFIVE_CIP_1200) \
: : "r" (addr), "r" (asid) : "memory")
-/*
- * _val is marked as "will be overwritten", so need to set it to 0
- * in the default case.
- */
-#define ALT_SVPBMT_SHIFT 61
-#define ALT_THEAD_MAE_SHIFT 59
-#define ALT_SVPBMT(_val, prot) \
-asm(ALTERNATIVE_2("li %0, 0\t\nnop", \
- "li %0, %1\t\nslli %0,%0,%3", 0, \
- RISCV_ISA_EXT_SVPBMT, CONFIG_RISCV_ISA_SVPBMT, \
- "li %0, %2\t\nslli %0,%0,%4", THEAD_VENDOR_ID, \
- ERRATA_THEAD_MAE, CONFIG_ERRATA_THEAD_MAE) \
- : "=r"(_val) \
- : "I"(prot##_SVPBMT >> ALT_SVPBMT_SHIFT), \
- "I"(prot##_THEAD >> ALT_THEAD_MAE_SHIFT), \
- "I"(ALT_SVPBMT_SHIFT), \
- "I"(ALT_THEAD_MAE_SHIFT))
-
-#ifdef CONFIG_ERRATA_THEAD_MAE
-/*
- * IO/NOCACHE memory types are handled together with svpbmt,
- * so on T-Head chips, check if no other memory type is set,
- * and set the non-0 PMA type if applicable.
- */
-#define ALT_THEAD_PMA(_val) \
-asm volatile(ALTERNATIVE( \
- __nops(7), \
- "li t3, %1\n\t" \
- "slli t3, t3, %3\n\t" \
- "and t3, %0, t3\n\t" \
- "bne t3, zero, 2f\n\t" \
- "li t3, %2\n\t" \
- "slli t3, t3, %3\n\t" \
- "or %0, %0, t3\n\t" \
- "2:", THEAD_VENDOR_ID, \
- ERRATA_THEAD_MAE, CONFIG_ERRATA_THEAD_MAE) \
- : "+r"(_val) \
- : "I"(_PAGE_MTMASK_THEAD >> ALT_THEAD_MAE_SHIFT), \
- "I"(_PAGE_PMA_THEAD >> ALT_THEAD_MAE_SHIFT), \
- "I"(ALT_THEAD_MAE_SHIFT) \
- : "t3")
-#else
-#define ALT_THEAD_PMA(_val)
-#endif
-
#define ALT_CMO_OP(_op, _start, _size, _cachesize) \
asm volatile(ALTERNATIVE( \
__nops(5), \
@@ -28,6 +28,9 @@
#define _PAGE_IO 0
#define _PAGE_MTMASK 0
+#define ALT_FIXUP_MT(_val)
+#define ALT_UNFIX_MT(_val)
+
#define pud_pfn(pud) (pmd_pfn((pmd_t){ pud }))
#define p4d_pfn(p4d) (pud_pfn((pud_t){ p4d }))
#define pgd_pfn(pgd) (p4d_pfn((p4d_t){ pgd }))
@@ -8,7 +8,7 @@
#include <linux/bits.h>
#include <linux/const.h>
-#include <asm/errata_list.h>
+#include <asm/alternative-macros.h>
extern bool pgtable_l4_enabled;
extern bool pgtable_l5_enabled;
@@ -109,6 +109,8 @@ enum napot_cont_order {
#define HUGE_MAX_HSTATE 2
#endif
+#ifdef CONFIG_RISCV_ISA_SVPBMT
+
/*
* [62:61] Svpbmt Memory Type definitions:
*
@@ -117,9 +119,9 @@ enum napot_cont_order {
* 10 - IO Non-cacheable, non-idempotent, strongly-ordered I/O memory
* 11 - Rsvd Reserved for future standard use
*/
-#define _PAGE_NOCACHE_SVPBMT (1UL << 61)
-#define _PAGE_IO_SVPBMT (1UL << 62)
-#define _PAGE_MTMASK_SVPBMT (_PAGE_NOCACHE_SVPBMT | _PAGE_IO_SVPBMT)
+#define _PAGE_NOCACHE (1UL << 61)
+#define _PAGE_IO (2UL << 61)
+#define _PAGE_MTMASK (3UL << 61)
/*
* [63:59] T-Head Memory Type definitions:
@@ -128,42 +130,66 @@ enum napot_cont_order {
* bit[61] B - Bufferable
* bit[60] SH - Shareable
* bit[59] Sec - Trustable
- * 00110 - NC Weakly-ordered, Non-cacheable, Bufferable, Shareable, Non-trustable
* 01110 - PMA Weakly-ordered, Cacheable, Bufferable, Shareable, Non-trustable
+ * 00110 - NC Weakly-ordered, Non-cacheable, Bufferable, Shareable, Non-trustable
* 10010 - IO Strongly-ordered, Non-cacheable, Non-bufferable, Shareable, Non-trustable
+ *
+ * ALT_FIXUP_MT translates Svpbmt memory types to XTheadMae memory types.
+ * Pseudocode operating on bits [63:60]:
+ * t0 = mt << 1
+ * if (t0 == 0)
+ * t0 |= 2
+ * t0 ^= 0x5
+ * mt ^= t0
+ *
+ * ALT_UNFIX_MT translates XTheadMae memory types to Svpbmt memory types.
+ * Pseudocode operating on bits [63:60]:
+ * t0 = mt & 0xd
+ * t0 ^= t0 >> 1
+ * mt ^= t0
*/
-#define _PAGE_PMA_THEAD ((1UL << 62) | (1UL << 61) | (1UL << 60))
-#define _PAGE_NOCACHE_THEAD ((1UL << 61) | (1UL << 60))
-#define _PAGE_IO_THEAD ((1UL << 63) | (1UL << 60))
-#define _PAGE_MTMASK_THEAD (_PAGE_PMA_THEAD | _PAGE_IO_THEAD | (1UL << 59))
-
-static inline u64 riscv_page_mtmask(void)
-{
- u64 val;
-
- ALT_SVPBMT(val, _PAGE_MTMASK);
- return val;
-}
-static inline u64 riscv_page_nocache(void)
-{
- u64 val;
+#define ALT_FIXUP_MT(_val) \
+ asm(ALTERNATIVE_2("addi t0, zero, 0x3\n\t" \
+ "slli t0, t0, 61\n\t" \
+ "not t0, t0\n\t" \
+ "and %0, %0, t0\n\t" \
+ "nop\n\t" \
+ "nop\n\t" \
+ "nop", \
+ __nops(7), \
+ 0, RISCV_ISA_EXT_SVPBMT, CONFIG_RISCV_ISA_SVPBMT, \
+ "srli t0, %0, 59\n\t" \
+ "seqz t1, t0\n\t" \
+ "slli t1, t1, 1\n\t" \
+ "or t0, t0, t1\n\t" \
+ "xori t0, t0, 0x5\n\t" \
+ "slli t0, t0, 60\n\t" \
+ "xor %0, %0, t0", \
+ THEAD_VENDOR_ID, ERRATA_THEAD_MAE, CONFIG_ERRATA_THEAD_MAE) \
+ : "+r" (_val) :: "t0", "t1")
+
+#define ALT_UNFIX_MT(_val) \
+ asm(ALTERNATIVE(__nops(6), \
+ "srli t0, %0, 60\n\t" \
+ "andi t0, t0, 0xd\n\t" \
+ "srli t1, t0, 1\n\t" \
+ "xor t0, t0, t1\n\t" \
+ "slli t0, t0, 60\n\t" \
+ "xor %0, %0, t0", \
+ THEAD_VENDOR_ID, ERRATA_THEAD_MAE, CONFIG_ERRATA_THEAD_MAE) \
+ : "+r" (_val) :: "t0", "t1")
- ALT_SVPBMT(val, _PAGE_NOCACHE);
- return val;
-}
+#else
-static inline u64 riscv_page_io(void)
-{
- u64 val;
+#define _PAGE_NOCACHE 0
+#define _PAGE_IO 0
+#define _PAGE_MTMASK 0
- ALT_SVPBMT(val, _PAGE_IO);
- return val;
-}
+#define ALT_FIXUP_MT(_val)
+#define ALT_UNFIX_MT(_val)
-#define _PAGE_NOCACHE riscv_page_nocache()
-#define _PAGE_IO riscv_page_io()
-#define _PAGE_MTMASK riscv_page_mtmask()
+#endif /* CONFIG_RISCV_ISA_SVPBMT */
static inline int pud_present(pud_t pud)
{
@@ -203,7 +229,11 @@ static inline void pud_clear(pud_t *pudp)
static inline pud_t pfn_pud(unsigned long pfn, pgprot_t prot)
{
- return __pud((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
+ pud_t pud = __pud((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
+
+ ALT_FIXUP_MT(pud);
+
+ return pud;
}
#define pud_pfn pud_pfn
@@ -244,11 +274,11 @@ static inline bool mm_pud_folded(struct mm_struct *mm)
static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot)
{
- unsigned long prot_val = pgprot_val(prot);
+ pmd_t pmd = __pmd((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
- ALT_THEAD_PMA(prot_val);
+ ALT_FIXUP_MT(pmd);
- return __pmd((pfn << _PAGE_PFN_SHIFT) | prot_val);
+ return pmd;
}
#define mk_pmd(page, prot) pfn_pmd(page_to_pfn(page), prot)
@@ -302,7 +332,11 @@ static inline void p4d_clear(p4d_t *p4d)
static inline p4d_t pfn_p4d(unsigned long pfn, pgprot_t prot)
{
- return __p4d((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
+ p4d_t p4d = __p4d((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
+
+ ALT_FIXUP_MT(p4d);
+
+ return p4d;
}
static inline unsigned long p4d_pfn(p4d_t p4d)
@@ -257,11 +257,11 @@ static inline void pmd_clear(pmd_t *pmdp)
static inline pgd_t pfn_pgd(unsigned long pfn, pgprot_t prot)
{
- unsigned long prot_val = pgprot_val(prot);
+ pgd_t pgd = __pgd((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
- ALT_THEAD_PMA(prot_val);
+ ALT_FIXUP_MT(pgd);
- return __pgd((pfn << _PAGE_PFN_SHIFT) | prot_val);
+ return pgd;
}
static inline unsigned long pmd_pfn(pmd_t pmd)
@@ -338,11 +338,11 @@ static inline unsigned long pte_pfn(pte_t pte)
/* Constructs a page table entry */
static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot)
{
- unsigned long prot_val = pgprot_val(prot);
+ pte_t pte = __pte((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
- ALT_THEAD_PMA(prot_val);
+ ALT_FIXUP_MT(pte);
- return __pte((pfn << _PAGE_PFN_SHIFT) | prot_val);
+ return pte;
}
#define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot)
@@ -489,9 +489,11 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
{
unsigned long newprot_val = pgprot_val(newprot);
- ALT_THEAD_PMA(newprot_val);
+ ALT_UNFIX_MT(pte);
+ pte = __pte((pte_val(pte) & _PAGE_CHG_MASK) | newprot_val);
+ ALT_FIXUP_MT(pte);
- return __pte((pte_val(pte) & _PAGE_CHG_MASK) | newprot_val);
+ return pte;
}
#define pgd_ERROR(e) \
@@ -142,7 +142,7 @@ static const struct prot_bits pte_bits[] = {
}, {
#endif
#ifdef CONFIG_RISCV_ISA_SVPBMT
- .mask = _PAGE_MTMASK_SVPBMT,
+ .mask = _PAGE_MTMASK,
.set = "MT(%s)",
.clear = " .. ",
}, {
@@ -218,10 +218,10 @@ static void dump_prot(struct pg_state *st)
if (pte_bits[i].mask == _PAGE_SOFT)
sprintf(s, pte_bits[i].set, val >> 8);
#ifdef CONFIG_RISCV_ISA_SVPBMT
- else if (pte_bits[i].mask == _PAGE_MTMASK_SVPBMT) {
- if (val == _PAGE_NOCACHE_SVPBMT)
+ else if (pte_bits[i].mask == _PAGE_MTMASK) {
+ if (val == _PAGE_NOCACHE)
sprintf(s, pte_bits[i].set, "NC");
- else if (val == _PAGE_IO_SVPBMT)
+ else if (val == _PAGE_IO)
sprintf(s, pte_bits[i].set, "IO");
else
sprintf(s, pte_bits[i].set, "??");
@@ -282,9 +282,12 @@ static void note_page(struct ptdump_state *pt_st, unsigned long addr,
int level, u64 val)
{
struct pg_state *st = container_of(pt_st, struct pg_state, ptdump);
- u64 pa = PFN_PHYS(pte_pfn(__pte(val)));
u64 prot = 0;
+ u64 pa;
+ ALT_UNFIX_MT(val);
+
+ pa = PFN_PHYS(pte_pfn(__pte(val)));
if (level >= 0)
prot = val & pg_level[level].mask;
Currently, Linux on RISC-V has three ways to specify the cacheability and ordering PMAs of a page: 1) Do nothing; assume the system is entirely cache-coherent and rely on the hardware for any ordering requirements 2) Use the page table bits specified by Svpbmt 3) Use the page table bits specified by XTheadMae To support all three methods, the kernel dynamically determines the definitions of the _PAGE_NOCACHE and _PAGE_IO fields. However, this alone is not sufficient, as XTheadMae uses a nonzero memory type value for normal memory pages. So the kernel has an additional alternative sequence (ALT_THEAD_PMA) to insert the correct memory type when writing page table entries. Some RISC-V platforms use a fourth method to specify the cacheability of a page of RAM: RAM is mapped to multiple physical address ranges, with each alias having a different set of statically-determined PMAs. Software selects the PMAs for a page by choosing a PFN from the corresponding physical address range. This strategy also requires applying a transformation when writing page table entries. Since these physical memory aliases should be invisible to the rest of the kernel, the opposite transformation must be applied when reading page table entries. However, with this last method of specifying PMAs, there is no inherent way to indicate the cacheability of a page in the pgprot_t value, since the PFN itself determines cacheability. One possible way is to reuse the PTE bits from Svpbmt, as Svpbmt is the standard extension. This requires the Svpbmt version of _PAGE_NOCACHE and _PAGE_IO to be available even when the CPU does not support the extension. It turns out that with some clever bit manipulation, it is just as efficient to transform all three Svpbmt memory type values to the corresponding XTheadMae values, as it is to check for and insert the one XTheadMae memory type value for normal memory. This allows the _PAGE_NOCACHE and _PAGE_IO definitions to be compile-time constants, and it centralizes all memory type handling to one set of ALTERNATIVE macros. For a kernel with both Svpbmt and XTheadMae enabled, this change reduces both kernel text size and the number of alternatives applied at boot. However, there are a couple of small costs. For platforms using the first method ("do nothing"), we must mask off the memory type bits when writing page table entries, whereas previously no action was needed. Second, when reading page table entries, the XTheadMae values must be transformed back to the Svpbmt values. This "unfix" operation is also needed for alias-based PMA selection, so both methods can use the same ALTERNATIVE. As a side effect, this change fixes the reporting of the NAPOT and memory type bits from ptdump on platforms with XTheadMae. Signed-off-by: Samuel Holland <samuel.holland@sifive.com> --- arch/riscv/Kconfig.errata | 1 + arch/riscv/include/asm/errata_list.h | 45 ----------- arch/riscv/include/asm/pgtable-32.h | 3 + arch/riscv/include/asm/pgtable-64.h | 108 ++++++++++++++++++--------- arch/riscv/include/asm/pgtable.h | 18 +++-- arch/riscv/mm/ptdump.c | 13 ++-- 6 files changed, 93 insertions(+), 95 deletions(-)