| Message ID | 20210407185918.371983-5-drjones@redhat.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | arm/arm64: Prepare for target-efi | expand |
On 07/04/2021 19:59, Andrew Jones wrote: > By providing a proper ioremap function, we can just rely on devices > calling it for each region they need (as they already do) instead of > mapping a big assumed I/O range. The persistent maps weirdness allows > us to call setup_vm after io_init. Why don't we just call setup_vm > before io_init, I hear you ask? Well, that's because tests like sieve > want to start with the MMU off and later call setup_vm, and all the > while have working I/O. Some unit tests are just really demanding... > > Signed-off-by: Andrew Jones <drjones@redhat.com> That's a very nice improvement! I wonder if any of the current calls to ioremap are for ranges big enough to allow for a sect map. However, this would be a performance improvent and something we can look at at some point in the future. Reviewed-by: Nikos Nikoleris <nikos.nikoleris@arm.com> > --- > lib/arm/asm/io.h | 6 ++++ > lib/arm/asm/mmu-api.h | 1 + > lib/arm/asm/mmu.h | 1 + > lib/arm/asm/page.h | 2 ++ > lib/arm/mmu.c | 82 ++++++++++++++++++++++++++++++++++++++----- > lib/arm64/asm/io.h | 6 ++++ > lib/arm64/asm/mmu.h | 1 + > lib/arm64/asm/page.h | 2 ++ > 8 files changed, 93 insertions(+), 8 deletions(-) > > diff --git a/lib/arm/asm/io.h b/lib/arm/asm/io.h > index ba3b0b2412ad..e4caa6ff5d1e 100644 > --- a/lib/arm/asm/io.h > +++ b/lib/arm/asm/io.h > @@ -77,6 +77,12 @@ static inline void __raw_writel(u32 val, volatile void __iomem *addr) > : "r" (val)); > } > > +#define ioremap ioremap > +static inline void __iomem *ioremap(phys_addr_t phys_addr, size_t size) > +{ > + return __ioremap(phys_addr, size); > +} > + > #define virt_to_phys virt_to_phys > static inline phys_addr_t virt_to_phys(const volatile void *x) > { > diff --git a/lib/arm/asm/mmu-api.h b/lib/arm/asm/mmu-api.h > index 05fc12b5afb8..b9a5a8f6b3c1 100644 > --- a/lib/arm/asm/mmu-api.h > +++ b/lib/arm/asm/mmu-api.h > @@ -17,6 +17,7 @@ extern void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset, > extern void mmu_set_range_ptes(pgd_t *pgtable, uintptr_t virt_offset, > phys_addr_t phys_start, phys_addr_t phys_end, > pgprot_t prot); > +extern void mmu_set_persistent_maps(pgd_t *pgtable); > extern pteval_t *mmu_get_pte(pgd_t *pgtable, uintptr_t vaddr); > extern void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr); > #endif > diff --git a/lib/arm/asm/mmu.h b/lib/arm/asm/mmu.h > index 122874b8aebe..d88a4f16df42 100644 > --- a/lib/arm/asm/mmu.h > +++ b/lib/arm/asm/mmu.h > @@ -12,6 +12,7 @@ > #define PTE_SHARED L_PTE_SHARED > #define PTE_AF PTE_EXT_AF > #define PTE_WBWA L_PTE_MT_WRITEALLOC > +#define PTE_UNCACHED L_PTE_MT_UNCACHED > > /* See B3.18.7 TLB maintenance operations */ > > diff --git a/lib/arm/asm/page.h b/lib/arm/asm/page.h > index 1fb5cd26ac66..8eb4a883808e 100644 > --- a/lib/arm/asm/page.h > +++ b/lib/arm/asm/page.h > @@ -47,5 +47,7 @@ typedef struct { pteval_t pgprot; } pgprot_t; > extern phys_addr_t __virt_to_phys(unsigned long addr); > extern unsigned long __phys_to_virt(phys_addr_t addr); > > +extern void *__ioremap(phys_addr_t phys_addr, size_t size); > + > #endif /* !__ASSEMBLY__ */ > #endif /* _ASMARM_PAGE_H_ */ > diff --git a/lib/arm/mmu.c b/lib/arm/mmu.c > index 15eef007f256..a7b7ae51afe3 100644 > --- a/lib/arm/mmu.c > +++ b/lib/arm/mmu.c > @@ -11,6 +11,7 @@ > #include <asm/mmu.h> > #include <asm/setup.h> > #include <asm/page.h> > +#include <asm/io.h> > > #include "alloc_page.h" > #include "vmalloc.h" > @@ -21,6 +22,57 @@ > > extern unsigned long etext; > > +#define MMU_MAX_PERSISTENT_MAPS 64 > + > +struct mmu_persistent_map { > + uintptr_t virt_offset; > + phys_addr_t phys_start; > + phys_addr_t phys_end; > + pgprot_t prot; > + bool sect; > +}; > + > +static struct mmu_persistent_map mmu_persistent_maps[MMU_MAX_PERSISTENT_MAPS]; > + > +static void > +mmu_set_persistent_range(uintptr_t virt_offset, phys_addr_t phys_start, > + phys_addr_t phys_end, pgprot_t prot, bool sect) > +{ > + int i; > + > + assert(phys_end); > + > + for (i = 0; i < MMU_MAX_PERSISTENT_MAPS; ++i) { > + if (!mmu_persistent_maps[i].phys_end) > + break; > + } > + assert(i < MMU_MAX_PERSISTENT_MAPS); > + > + mmu_persistent_maps[i] = (struct mmu_persistent_map){ > + .virt_offset = virt_offset, > + .phys_start = phys_start, > + .phys_end = phys_end, > + .prot = prot, > + .sect = sect, > + }; > +} > + > +void mmu_set_persistent_maps(pgd_t *pgtable) > +{ > + struct mmu_persistent_map *map; > + > + for (map = &mmu_persistent_maps[0]; map->phys_end; ++map) { > + if (map->sect) > + mmu_set_range_sect(pgtable, map->virt_offset, > + map->phys_start, map->phys_end, > + map->prot); > + else > + mmu_set_range_ptes(pgtable, map->virt_offset, > + map->phys_start, map->phys_end, > + map->prot); > + } > +} > + > pgd_t *mmu_idmap; > > /* CPU 0 starts with disabled MMU */ > @@ -157,7 +209,6 @@ void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset, > void *setup_mmu(phys_addr_t phys_end) > { > uintptr_t code_end = (uintptr_t)&etext; > - struct mem_region *r; > > /* 0G-1G = I/O, 1G-3G = identity, 3G-4G = vmalloc */ > if (phys_end > (3ul << 30)) > @@ -172,13 +223,6 @@ void *setup_mmu(phys_addr_t phys_end) > > mmu_idmap = alloc_page(); > > - for (r = mem_regions; r->end; ++r) { > - if (!(r->flags & MR_F_IO)) > - continue; > - mmu_set_range_sect(mmu_idmap, r->start, r->start, r->end, > - __pgprot(PMD_SECT_UNCACHED | PMD_SECT_USER)); > - } > - > /* armv8 requires code shared between EL1 and EL0 to be read-only */ > mmu_set_range_ptes(mmu_idmap, PHYS_OFFSET, > PHYS_OFFSET, code_end, > @@ -188,10 +232,32 @@ void *setup_mmu(phys_addr_t phys_end) > code_end, phys_end, > __pgprot(PTE_WBWA | PTE_USER)); > > + mmu_set_persistent_maps(mmu_idmap); > + > mmu_enable(mmu_idmap); > return mmu_idmap; > } > > +void __iomem *__ioremap(phys_addr_t phys_addr, size_t size) > +{ > + phys_addr_t paddr_aligned = phys_addr & PAGE_MASK; > + phys_addr_t paddr_end = PAGE_ALIGN(phys_addr + size); > + pgprot_t prot = __pgprot(PTE_UNCACHED | PTE_USER); > + > + assert(sizeof(long) == 8 || !(phys_addr >> 32)); > + > + mmu_set_persistent_range(paddr_aligned, paddr_aligned, paddr_end, > + prot, false); > + > + if (mmu_enabled()) { > + pgd_t *pgtable = current_thread_info()->pgtable; > + mmu_set_range_ptes(pgtable, paddr_aligned, paddr_aligned, > + paddr_end, prot); > + } > + > + return (void __iomem *)(unsigned long)phys_addr; > +} > + > phys_addr_t __virt_to_phys(unsigned long addr) > { > if (mmu_enabled()) { > diff --git a/lib/arm64/asm/io.h b/lib/arm64/asm/io.h > index e0a03b250d5b..be19f471c0fa 100644 > --- a/lib/arm64/asm/io.h > +++ b/lib/arm64/asm/io.h > @@ -71,6 +71,12 @@ static inline u64 __raw_readq(const volatile void __iomem *addr) > return val; > } > > +#define ioremap ioremap > +static inline void __iomem *ioremap(phys_addr_t phys_addr, size_t size) > +{ > + return __ioremap(phys_addr, size); > +} > + > #define virt_to_phys virt_to_phys > static inline phys_addr_t virt_to_phys(const volatile void *x) > { > diff --git a/lib/arm64/asm/mmu.h b/lib/arm64/asm/mmu.h > index 72d75eafc882..72371b2d9fe3 100644 > --- a/lib/arm64/asm/mmu.h > +++ b/lib/arm64/asm/mmu.h > @@ -8,6 +8,7 @@ > #include <asm/barrier.h> > > #define PMD_SECT_UNCACHED PMD_ATTRINDX(MT_DEVICE_nGnRE) > +#define PTE_UNCACHED PTE_ATTRINDX(MT_DEVICE_nGnRE) > #define PTE_WBWA PTE_ATTRINDX(MT_NORMAL) > > static inline void flush_tlb_all(void) > diff --git a/lib/arm64/asm/page.h b/lib/arm64/asm/page.h > index ae4484b22114..d0fac6ea563d 100644 > --- a/lib/arm64/asm/page.h > +++ b/lib/arm64/asm/page.h > @@ -72,5 +72,7 @@ typedef struct { pteval_t pgprot; } pgprot_t; > extern phys_addr_t __virt_to_phys(unsigned long addr); > extern unsigned long __phys_to_virt(phys_addr_t addr); > > +extern void *__ioremap(phys_addr_t phys_addr, size_t size); > + > #endif /* !__ASSEMBLY__ */ > #endif /* _ASMARM64_PAGE_H_ */ >
Hi Drew, On 4/7/21 7:59 PM, Andrew Jones wrote: > By providing a proper ioremap function, we can just rely on devices > calling it for each region they need (as they already do) instead of > mapping a big assumed I/O range. The persistent maps weirdness allows > us to call setup_vm after io_init. Why don't we just call setup_vm > before io_init, I hear you ask? Well, that's because tests like sieve > want to start with the MMU off and later call setup_vm, and all the > while have working I/O. Some unit tests are just really demanding... > > Signed-off-by: Andrew Jones <drjones@redhat.com> > --- > lib/arm/asm/io.h | 6 ++++ > lib/arm/asm/mmu-api.h | 1 + > lib/arm/asm/mmu.h | 1 + > lib/arm/asm/page.h | 2 ++ > lib/arm/mmu.c | 82 ++++++++++++++++++++++++++++++++++++++----- > lib/arm64/asm/io.h | 6 ++++ > lib/arm64/asm/mmu.h | 1 + > lib/arm64/asm/page.h | 2 ++ > 8 files changed, 93 insertions(+), 8 deletions(-) > > diff --git a/lib/arm/asm/io.h b/lib/arm/asm/io.h > index ba3b0b2412ad..e4caa6ff5d1e 100644 > --- a/lib/arm/asm/io.h > +++ b/lib/arm/asm/io.h > @@ -77,6 +77,12 @@ static inline void __raw_writel(u32 val, volatile void __iomem *addr) > : "r" (val)); > } > > +#define ioremap ioremap > +static inline void __iomem *ioremap(phys_addr_t phys_addr, size_t size) > +{ > + return __ioremap(phys_addr, size); > +} > + > #define virt_to_phys virt_to_phys > static inline phys_addr_t virt_to_phys(const volatile void *x) > { > diff --git a/lib/arm/asm/mmu-api.h b/lib/arm/asm/mmu-api.h > index 05fc12b5afb8..b9a5a8f6b3c1 100644 > --- a/lib/arm/asm/mmu-api.h > +++ b/lib/arm/asm/mmu-api.h > @@ -17,6 +17,7 @@ extern void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset, > extern void mmu_set_range_ptes(pgd_t *pgtable, uintptr_t virt_offset, > phys_addr_t phys_start, phys_addr_t phys_end, > pgprot_t prot); > +extern void mmu_set_persistent_maps(pgd_t *pgtable); > extern pteval_t *mmu_get_pte(pgd_t *pgtable, uintptr_t vaddr); > extern void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr); > #endif > diff --git a/lib/arm/asm/mmu.h b/lib/arm/asm/mmu.h > index 122874b8aebe..d88a4f16df42 100644 > --- a/lib/arm/asm/mmu.h > +++ b/lib/arm/asm/mmu.h > @@ -12,6 +12,7 @@ > #define PTE_SHARED L_PTE_SHARED > #define PTE_AF PTE_EXT_AF > #define PTE_WBWA L_PTE_MT_WRITEALLOC > +#define PTE_UNCACHED L_PTE_MT_UNCACHED > > /* See B3.18.7 TLB maintenance operations */ > > diff --git a/lib/arm/asm/page.h b/lib/arm/asm/page.h > index 1fb5cd26ac66..8eb4a883808e 100644 > --- a/lib/arm/asm/page.h > +++ b/lib/arm/asm/page.h > @@ -47,5 +47,7 @@ typedef struct { pteval_t pgprot; } pgprot_t; > extern phys_addr_t __virt_to_phys(unsigned long addr); > extern unsigned long __phys_to_virt(phys_addr_t addr); > > +extern void *__ioremap(phys_addr_t phys_addr, size_t size); > + > #endif /* !__ASSEMBLY__ */ > #endif /* _ASMARM_PAGE_H_ */ > diff --git a/lib/arm/mmu.c b/lib/arm/mmu.c > index 15eef007f256..a7b7ae51afe3 100644 > --- a/lib/arm/mmu.c > +++ b/lib/arm/mmu.c > @@ -11,6 +11,7 @@ > #include <asm/mmu.h> > #include <asm/setup.h> > #include <asm/page.h> > +#include <asm/io.h> > > #include "alloc_page.h" > #include "vmalloc.h" > @@ -21,6 +22,57 @@ > > extern unsigned long etext; > > +#define MMU_MAX_PERSISTENT_MAPS 64 > + > +struct mmu_persistent_map { > + uintptr_t virt_offset; > + phys_addr_t phys_start; > + phys_addr_t phys_end; > + pgprot_t prot; > + bool sect; > +}; > + > +static struct mmu_persistent_map mmu_persistent_maps[MMU_MAX_PERSISTENT_MAPS]; > + > +static void > +mmu_set_persistent_range(uintptr_t virt_offset, phys_addr_t phys_start, > + phys_addr_t phys_end, pgprot_t prot, bool sect) > +{ > + int i; > + > + assert(phys_end); > + > + for (i = 0; i < MMU_MAX_PERSISTENT_MAPS; ++i) { > + if (!mmu_persistent_maps[i].phys_end) > + break; > + } > + assert(i < MMU_MAX_PERSISTENT_MAPS); > + > + mmu_persistent_maps[i] = (struct mmu_persistent_map){ > + .virt_offset = virt_offset, > + .phys_start = phys_start, > + .phys_end = phys_end, > + .prot = prot, > + .sect = sect, > + }; > +} > + > +void mmu_set_persistent_maps(pgd_t *pgtable) > +{ > + struct mmu_persistent_map *map; > + > + for (map = &mmu_persistent_maps[0]; map->phys_end; ++map) { > + if (map->sect) > + mmu_set_range_sect(pgtable, map->virt_offset, > + map->phys_start, map->phys_end, > + map->prot); > + else > + mmu_set_range_ptes(pgtable, map->virt_offset, > + map->phys_start, map->phys_end, > + map->prot); > + } > +} I assume the purpose of all of this machinery is to add mappings to idmap that were created before setup_mmu(). Or are you planning to use it for something else? Why not allocate the idmap in __ioremap (if it's NULL) and add entries to it in that function? Then setup_mmu() can allocate the idmap only if it's NULL, and the mappings added by __ioremap would still be there. Thanks, Alex > + > pgd_t *mmu_idmap; > > /* CPU 0 starts with disabled MMU */ > @@ -157,7 +209,6 @@ void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset, > void *setup_mmu(phys_addr_t phys_end) > { > uintptr_t code_end = (uintptr_t)&etext; > - struct mem_region *r; > > /* 0G-1G = I/O, 1G-3G = identity, 3G-4G = vmalloc */ > if (phys_end > (3ul << 30)) > @@ -172,13 +223,6 @@ void *setup_mmu(phys_addr_t phys_end) > > mmu_idmap = alloc_page(); > > - for (r = mem_regions; r->end; ++r) { > - if (!(r->flags & MR_F_IO)) > - continue; > - mmu_set_range_sect(mmu_idmap, r->start, r->start, r->end, > - __pgprot(PMD_SECT_UNCACHED | PMD_SECT_USER)); > - } > - > /* armv8 requires code shared between EL1 and EL0 to be read-only */ > mmu_set_range_ptes(mmu_idmap, PHYS_OFFSET, > PHYS_OFFSET, code_end, > @@ -188,10 +232,32 @@ void *setup_mmu(phys_addr_t phys_end) > code_end, phys_end, > __pgprot(PTE_WBWA | PTE_USER)); > > + mmu_set_persistent_maps(mmu_idmap); > + > mmu_enable(mmu_idmap); > return mmu_idmap; > } > > +void __iomem *__ioremap(phys_addr_t phys_addr, size_t size) > +{ > + phys_addr_t paddr_aligned = phys_addr & PAGE_MASK; > + phys_addr_t paddr_end = PAGE_ALIGN(phys_addr + size); > + pgprot_t prot = __pgprot(PTE_UNCACHED | PTE_USER); > + > + assert(sizeof(long) == 8 || !(phys_addr >> 32)); > + > + mmu_set_persistent_range(paddr_aligned, paddr_aligned, paddr_end, > + prot, false); > + > + if (mmu_enabled()) { > + pgd_t *pgtable = current_thread_info()->pgtable; > + mmu_set_range_ptes(pgtable, paddr_aligned, paddr_aligned, > + paddr_end, prot); > + } > + > + return (void __iomem *)(unsigned long)phys_addr; > +} > + > phys_addr_t __virt_to_phys(unsigned long addr) > { > if (mmu_enabled()) { > diff --git a/lib/arm64/asm/io.h b/lib/arm64/asm/io.h > index e0a03b250d5b..be19f471c0fa 100644 > --- a/lib/arm64/asm/io.h > +++ b/lib/arm64/asm/io.h > @@ -71,6 +71,12 @@ static inline u64 __raw_readq(const volatile void __iomem *addr) > return val; > } > > +#define ioremap ioremap > +static inline void __iomem *ioremap(phys_addr_t phys_addr, size_t size) > +{ > + return __ioremap(phys_addr, size); > +} > + > #define virt_to_phys virt_to_phys > static inline phys_addr_t virt_to_phys(const volatile void *x) > { > diff --git a/lib/arm64/asm/mmu.h b/lib/arm64/asm/mmu.h > index 72d75eafc882..72371b2d9fe3 100644 > --- a/lib/arm64/asm/mmu.h > +++ b/lib/arm64/asm/mmu.h > @@ -8,6 +8,7 @@ > #include <asm/barrier.h> > > #define PMD_SECT_UNCACHED PMD_ATTRINDX(MT_DEVICE_nGnRE) > +#define PTE_UNCACHED PTE_ATTRINDX(MT_DEVICE_nGnRE) > #define PTE_WBWA PTE_ATTRINDX(MT_NORMAL) > > static inline void flush_tlb_all(void) > diff --git a/lib/arm64/asm/page.h b/lib/arm64/asm/page.h > index ae4484b22114..d0fac6ea563d 100644 > --- a/lib/arm64/asm/page.h > +++ b/lib/arm64/asm/page.h > @@ -72,5 +72,7 @@ typedef struct { pteval_t pgprot; } pgprot_t; > extern phys_addr_t __virt_to_phys(unsigned long addr); > extern unsigned long __phys_to_virt(phys_addr_t addr); > > +extern void *__ioremap(phys_addr_t phys_addr, size_t size); > + > #endif /* !__ASSEMBLY__ */ > #endif /* _ASMARM64_PAGE_H_ */
On Wed, Apr 14, 2021 at 04:42:17PM +0100, Alexandru Elisei wrote: > On 4/7/21 7:59 PM, Andrew Jones wrote: > > +void mmu_set_persistent_maps(pgd_t *pgtable) > > +{ > > + struct mmu_persistent_map *map; > > + > > + for (map = &mmu_persistent_maps[0]; map->phys_end; ++map) { > > + if (map->sect) > > + mmu_set_range_sect(pgtable, map->virt_offset, > > + map->phys_start, map->phys_end, > > + map->prot); > > + else > > + mmu_set_range_ptes(pgtable, map->virt_offset, > > + map->phys_start, map->phys_end, > > + map->prot); > > + } > > +} > > I assume the purpose of all of this machinery is to add mappings to idmap that > were created before setup_mmu(). Or are you planning to use it for something else? > > Why not allocate the idmap in __ioremap (if it's NULL) and add entries to it in > that function? Then setup_mmu() can allocate the idmap only if it's NULL, and the > mappings added by __ioremap would still be there. > Hi Alex, I like your suggestion and will implement it that way for v2. If we ever do need these mappings for anything else, then we can revisit this, possibly stashing the mappings at the same time we add them to the idmap. Thanks, drew
diff --git a/lib/arm/asm/io.h b/lib/arm/asm/io.h index ba3b0b2412ad..e4caa6ff5d1e 100644 --- a/lib/arm/asm/io.h +++ b/lib/arm/asm/io.h @@ -77,6 +77,12 @@ static inline void __raw_writel(u32 val, volatile void __iomem *addr) : "r" (val)); } +#define ioremap ioremap +static inline void __iomem *ioremap(phys_addr_t phys_addr, size_t size) +{ + return __ioremap(phys_addr, size); +} + #define virt_to_phys virt_to_phys static inline phys_addr_t virt_to_phys(const volatile void *x) { diff --git a/lib/arm/asm/mmu-api.h b/lib/arm/asm/mmu-api.h index 05fc12b5afb8..b9a5a8f6b3c1 100644 --- a/lib/arm/asm/mmu-api.h +++ b/lib/arm/asm/mmu-api.h @@ -17,6 +17,7 @@ extern void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset, extern void mmu_set_range_ptes(pgd_t *pgtable, uintptr_t virt_offset, phys_addr_t phys_start, phys_addr_t phys_end, pgprot_t prot); +extern void mmu_set_persistent_maps(pgd_t *pgtable); extern pteval_t *mmu_get_pte(pgd_t *pgtable, uintptr_t vaddr); extern void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr); #endif diff --git a/lib/arm/asm/mmu.h b/lib/arm/asm/mmu.h index 122874b8aebe..d88a4f16df42 100644 --- a/lib/arm/asm/mmu.h +++ b/lib/arm/asm/mmu.h @@ -12,6 +12,7 @@ #define PTE_SHARED L_PTE_SHARED #define PTE_AF PTE_EXT_AF #define PTE_WBWA L_PTE_MT_WRITEALLOC +#define PTE_UNCACHED L_PTE_MT_UNCACHED /* See B3.18.7 TLB maintenance operations */ diff --git a/lib/arm/asm/page.h b/lib/arm/asm/page.h index 1fb5cd26ac66..8eb4a883808e 100644 --- a/lib/arm/asm/page.h +++ b/lib/arm/asm/page.h @@ -47,5 +47,7 @@ typedef struct { pteval_t pgprot; } pgprot_t; extern phys_addr_t __virt_to_phys(unsigned long addr); extern unsigned long __phys_to_virt(phys_addr_t addr); +extern void *__ioremap(phys_addr_t phys_addr, size_t size); + #endif /* !__ASSEMBLY__ */ #endif /* _ASMARM_PAGE_H_ */ diff --git a/lib/arm/mmu.c b/lib/arm/mmu.c index 15eef007f256..a7b7ae51afe3 100644 --- a/lib/arm/mmu.c +++ b/lib/arm/mmu.c @@ -11,6 +11,7 @@ #include <asm/mmu.h> #include <asm/setup.h> #include <asm/page.h> +#include <asm/io.h> #include "alloc_page.h" #include "vmalloc.h" @@ -21,6 +22,57 @@ extern unsigned long etext; +#define MMU_MAX_PERSISTENT_MAPS 64 + +struct mmu_persistent_map { + uintptr_t virt_offset; + phys_addr_t phys_start; + phys_addr_t phys_end; + pgprot_t prot; + bool sect; +}; + +static struct mmu_persistent_map mmu_persistent_maps[MMU_MAX_PERSISTENT_MAPS]; + +static void +mmu_set_persistent_range(uintptr_t virt_offset, phys_addr_t phys_start, + phys_addr_t phys_end, pgprot_t prot, bool sect) +{ + int i; + + assert(phys_end); + + for (i = 0; i < MMU_MAX_PERSISTENT_MAPS; ++i) { + if (!mmu_persistent_maps[i].phys_end) + break; + } + assert(i < MMU_MAX_PERSISTENT_MAPS); + + mmu_persistent_maps[i] = (struct mmu_persistent_map){ + .virt_offset = virt_offset, + .phys_start = phys_start, + .phys_end = phys_end, + .prot = prot, + .sect = sect, + }; +} + +void mmu_set_persistent_maps(pgd_t *pgtable) +{ + struct mmu_persistent_map *map; + + for (map = &mmu_persistent_maps[0]; map->phys_end; ++map) { + if (map->sect) + mmu_set_range_sect(pgtable, map->virt_offset, + map->phys_start, map->phys_end, + map->prot); + else + mmu_set_range_ptes(pgtable, map->virt_offset, + map->phys_start, map->phys_end, + map->prot); + } +} + pgd_t *mmu_idmap; /* CPU 0 starts with disabled MMU */ @@ -157,7 +209,6 @@ void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset, void *setup_mmu(phys_addr_t phys_end) { uintptr_t code_end = (uintptr_t)&etext; - struct mem_region *r; /* 0G-1G = I/O, 1G-3G = identity, 3G-4G = vmalloc */ if (phys_end > (3ul << 30)) @@ -172,13 +223,6 @@ void *setup_mmu(phys_addr_t phys_end) mmu_idmap = alloc_page(); - for (r = mem_regions; r->end; ++r) { - if (!(r->flags & MR_F_IO)) - continue; - mmu_set_range_sect(mmu_idmap, r->start, r->start, r->end, - __pgprot(PMD_SECT_UNCACHED | PMD_SECT_USER)); - } - /* armv8 requires code shared between EL1 and EL0 to be read-only */ mmu_set_range_ptes(mmu_idmap, PHYS_OFFSET, PHYS_OFFSET, code_end, @@ -188,10 +232,32 @@ void *setup_mmu(phys_addr_t phys_end) code_end, phys_end, __pgprot(PTE_WBWA | PTE_USER)); + mmu_set_persistent_maps(mmu_idmap); + mmu_enable(mmu_idmap); return mmu_idmap; } +void __iomem *__ioremap(phys_addr_t phys_addr, size_t size) +{ + phys_addr_t paddr_aligned = phys_addr & PAGE_MASK; + phys_addr_t paddr_end = PAGE_ALIGN(phys_addr + size); + pgprot_t prot = __pgprot(PTE_UNCACHED | PTE_USER); + + assert(sizeof(long) == 8 || !(phys_addr >> 32)); + + mmu_set_persistent_range(paddr_aligned, paddr_aligned, paddr_end, + prot, false); + + if (mmu_enabled()) { + pgd_t *pgtable = current_thread_info()->pgtable; + mmu_set_range_ptes(pgtable, paddr_aligned, paddr_aligned, + paddr_end, prot); + } + + return (void __iomem *)(unsigned long)phys_addr; +} + phys_addr_t __virt_to_phys(unsigned long addr) { if (mmu_enabled()) { diff --git a/lib/arm64/asm/io.h b/lib/arm64/asm/io.h index e0a03b250d5b..be19f471c0fa 100644 --- a/lib/arm64/asm/io.h +++ b/lib/arm64/asm/io.h @@ -71,6 +71,12 @@ static inline u64 __raw_readq(const volatile void __iomem *addr) return val; } +#define ioremap ioremap +static inline void __iomem *ioremap(phys_addr_t phys_addr, size_t size) +{ + return __ioremap(phys_addr, size); +} + #define virt_to_phys virt_to_phys static inline phys_addr_t virt_to_phys(const volatile void *x) { diff --git a/lib/arm64/asm/mmu.h b/lib/arm64/asm/mmu.h index 72d75eafc882..72371b2d9fe3 100644 --- a/lib/arm64/asm/mmu.h +++ b/lib/arm64/asm/mmu.h @@ -8,6 +8,7 @@ #include <asm/barrier.h> #define PMD_SECT_UNCACHED PMD_ATTRINDX(MT_DEVICE_nGnRE) +#define PTE_UNCACHED PTE_ATTRINDX(MT_DEVICE_nGnRE) #define PTE_WBWA PTE_ATTRINDX(MT_NORMAL) static inline void flush_tlb_all(void) diff --git a/lib/arm64/asm/page.h b/lib/arm64/asm/page.h index ae4484b22114..d0fac6ea563d 100644 --- a/lib/arm64/asm/page.h +++ b/lib/arm64/asm/page.h @@ -72,5 +72,7 @@ typedef struct { pteval_t pgprot; } pgprot_t; extern phys_addr_t __virt_to_phys(unsigned long addr); extern unsigned long __phys_to_virt(phys_addr_t addr); +extern void *__ioremap(phys_addr_t phys_addr, size_t size); + #endif /* !__ASSEMBLY__ */ #endif /* _ASMARM64_PAGE_H_ */
By providing a proper ioremap function, we can just rely on devices calling it for each region they need (as they already do) instead of mapping a big assumed I/O range. The persistent maps weirdness allows us to call setup_vm after io_init. Why don't we just call setup_vm before io_init, I hear you ask? Well, that's because tests like sieve want to start with the MMU off and later call setup_vm, and all the while have working I/O. Some unit tests are just really demanding... Signed-off-by: Andrew Jones <drjones@redhat.com> --- lib/arm/asm/io.h | 6 ++++ lib/arm/asm/mmu-api.h | 1 + lib/arm/asm/mmu.h | 1 + lib/arm/asm/page.h | 2 ++ lib/arm/mmu.c | 82 ++++++++++++++++++++++++++++++++++++++----- lib/arm64/asm/io.h | 6 ++++ lib/arm64/asm/mmu.h | 1 + lib/arm64/asm/page.h | 2 ++ 8 files changed, 93 insertions(+), 8 deletions(-)