@@ -7,11 +7,15 @@
#include <linux/kprobes.h>
+#include <asm/pgtable-types.h>
+
#ifdef CONFIG_REPLICATE_KTEXT
void ktext_replication_init(void);
void __kprobes ktext_replication_patch(u32 *tp, __le32 insn);
void ktext_replication_patch_alternative(__le32 *src, int nr_inst);
+void ktext_replication_set_swapper_pgd(pgd_t *pgdp, pgd_t pgd);
+void ktext_replication_init_tramp(void);
#else
@@ -27,6 +31,14 @@ static inline void ktext_replication_patch_alternative(__le32 *src, int nr_inst)
{
}
+static inline void ktext_replication_set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
+{
+}
+
+static inline void ktext_replication_init_tramp(void)
+{
+}
+
#endif
#endif
@@ -14,6 +14,7 @@
#include <asm/cacheflush.h>
#include <asm/ktext.h>
#include <asm/memory.h>
+#include <asm/pgalloc.h>
struct pgtables *pgtables[MAX_NUMNODES] = {
[0 ... MAX_NUMNODES - 1] = &pgtable_node0,
@@ -97,7 +98,7 @@ void ktext_replication_patch_alternative(__le32 *src, int nr_inst)
}
}
-/* Allocate memory for the replicated kernel texts. */
+/* Allocate page tables and memory for the replicated kernel texts. */
void __init ktext_replication_init(void)
{
size_t size = _etext - _stext;
@@ -128,5 +129,44 @@ void __init ktext_replication_init(void)
memcpy(kernel_texts[nid], _stext, size);
caches_clean_inval_pou((u64)kernel_texts[nid],
(u64)kernel_texts[nid] + size);
+
+ /* Allocate the pagetables for this node */
+ pgtables[nid] = memblock_alloc_node(sizeof(*pgtables[0]),
+ PGD_SIZE, nid);
+
+ /* Copy initial swapper page directory */
+ memcpy(pgtables[nid]->swapper_pg_dir, swapper_pg_dir, PGD_SIZE);
+ }
+}
+
+void ktext_replication_set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
+{
+ unsigned long idx = pgdp - swapper_pg_dir;
+ int nid;
+
+ if (WARN_ON_ONCE(idx >= PTRS_PER_PGD) ||
+ WARN_ON_ONCE(idx == pgd_index((phys_addr_t)KERNEL_START)))
+ return;
+
+ for_each_node(nid) {
+ if (pgtables[nid]->swapper_pg_dir == swapper_pg_dir)
+ continue;
+
+ WRITE_ONCE(pgtables[nid]->swapper_pg_dir[idx], pgd);
+ }
+}
+
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+void __init ktext_replication_init_tramp(void)
+{
+ int nid;
+
+ for_each_node(nid) {
+ /* Nothing to do for node 0 */
+ if (pgtables[nid]->tramp_pg_dir == tramp_pg_dir)
+ continue;
+
+ memcpy(pgtables[nid]->tramp_pg_dir, tramp_pg_dir, PGD_SIZE);
}
}
+#endif
@@ -31,6 +31,7 @@
#include <asm/fixmap.h>
#include <asm/kasan.h>
#include <asm/kernel-pgtable.h>
+#include <asm/ktext.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <linux/sizes.h>
@@ -81,6 +82,7 @@ void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
pgd_t *fixmap_pgdp;
spin_lock(&swapper_pgdir_lock);
+ ktext_replication_set_swapper_pgd(pgdp, pgd);
fixmap_pgdp = pgd_set_fixmap(__pa_symbol(pgdp));
WRITE_ONCE(*fixmap_pgdp, pgd);
/*
@@ -695,6 +697,9 @@ static int __init map_entry_trampoline(void)
__set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
pa_start + i * PAGE_SIZE, PAGE_KERNEL_RO);
+ /* Copy trampoline page tables to other numa nodes */
+ ktext_replication_init_tramp();
+
return 0;
}
core_initcall(map_entry_trampoline);
Allocate the level 0 page tables for the per-node kernel text replication, but copy all level 0 table entries from the NUMA node 0 table. Therefore, for the time being, each node's level 0 page tables will contain identical entries, and thus other nodes will continue to use the node 0 kernel text. Since the level 0 page tables can be updated at runtime to add entries for vmalloc and module space, propagate these updates to the other swapper page tables. The exception is if we see an update for the level 0 entry which points to the kernel mapping. We also need to setup a copy of the trampoline page tables as well, as the assembly code relies on the two page tables being a fixed offset apart. Signed-off-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk> --- arch/arm64/include/asm/ktext.h | 12 ++++++++++ arch/arm64/mm/ktext.c | 42 +++++++++++++++++++++++++++++++++- arch/arm64/mm/mmu.c | 5 ++++ 3 files changed, 58 insertions(+), 1 deletion(-)