From patchwork Wed Mar 13 12:57:07 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pingfan Liu X-Patchwork-Id: 13591391 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from bombadil.infradead.org (bombadil.infradead.org [198.137.202.133]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.lore.kernel.org (Postfix) with ESMTPS id 385F4C54E58 for ; Wed, 13 Mar 2024 13:00:53 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=lists.infradead.org; s=bombadil.20210309; h=Sender: Content-Transfer-Encoding:Content-Type:List-Subscribe:List-Help:List-Post: List-Archive:List-Unsubscribe:List-Id:MIME-Version:References:In-Reply-To: Message-ID:Date:Subject:Cc:To:From:Reply-To:Content-ID:Content-Description: Resent-Date:Resent-From:Resent-Sender:Resent-To:Resent-Cc:Resent-Message-ID: List-Owner; bh=19inU/4PpJvEraAK6uqO6UHjKzKuH31inGUbih12M6Q=; b=OJFQeSSGzr8EG0 uKQ2V1fdl7ZCm/1qvySD/nZEmeb/O0IEC3i1+CafR2F9jid8GOZmvrBJs6f6S0+Z7Z+MjOgtTqeL0 a2iXMjt1ftgeUv7MOBpSA618qq2awrnhpsjxL6sWpEe27bjkIVfZ7zLuHreq3XzDkyvB3hU38h6Ee 56jVyRk40wtpCv4penOK4zzoU8hMy6Ne389FuLwbM87eIcioEMZQZT6bkthSmVkVkP9Sq1fWAflLX zKyMIjXLvULBJl2c97HePhVJCSjV+Z51WHmKwQ+VbOiPw7ooQeycsrSiZMis1djwDt9ym4oiFv31u Mvv/wh3na6vH1cl6a9vA==; Received: from localhost ([::1] helo=bombadil.infradead.org) by bombadil.infradead.org with esmtp (Exim 4.97.1 #2 (Red Hat Linux)) id 1rkODi-0000000A9Vp-1UPX; Wed, 13 Mar 2024 13:00:34 +0000 Received: from us-smtp-delivery-124.mimecast.com ([170.10.133.124]) by bombadil.infradead.org with esmtps (Exim 4.97.1 #2 (Red Hat Linux)) id 1rkODb-0000000A9Ra-0rwg for linux-arm-kernel@lists.infradead.org; Wed, 13 Mar 2024 13:00:30 +0000 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=redhat.com; s=mimecast20190719; t=1710334826; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=EwPhW7O+Fl1hLMWiwjIgjkNT9KeW91Gso2Lq5mgSCuY=; b=EyFscRwZx2m5hAGcpQFBQSOjvzRhaTcXa8OKBaYGkrK7jZLjpn059ihBQL10px3zJ81V0k Vk82WDDzd72ikdguHBR9leg2UJxCZf3QYpkneAfMCBb6RyHFSjtPUzt1lslHZpzCemjO2B SkC3NHOq9EAgzjqzQcgk7C7M8aLofNM= Received: from mimecast-mx02.redhat.com (mimecast-mx02.redhat.com [66.187.233.88]) by relay.mimecast.com with ESMTP with STARTTLS (version=TLSv1.3, cipher=TLS_AES_256_GCM_SHA384) id us-mta-695-Z34CSwRxP4iv9XxebPKaZg-1; Wed, 13 Mar 2024 08:57:53 -0400 X-MC-Unique: Z34CSwRxP4iv9XxebPKaZg-1 Received: from smtp.corp.redhat.com (int-mx05.intmail.prod.int.rdu2.redhat.com [10.11.54.5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by mimecast-mx02.redhat.com (Postfix) with ESMTPS id 5D5A9101A58E; Wed, 13 Mar 2024 12:57:53 +0000 (UTC) Received: from fedora.redhat.com (unknown [10.72.112.95]) by smtp.corp.redhat.com (Postfix) with ESMTP id D6A5B10E47; Wed, 13 Mar 2024 12:57:50 +0000 (UTC) From: Pingfan Liu To: linux-arm-kernel@lists.infradead.org Cc: Pingfan Liu , Ard Biesheuvel , Catalin Marinas , Will Deacon , Mark Rutland Subject: [PATCH 09/10] arm64: head: Use __create_pgd_mapping_locked() to serve the creation of pgtable Date: Wed, 13 Mar 2024 20:57:07 +0800 Message-ID: <20240313125711.20651-10-piliu@redhat.com> In-Reply-To: <20240313125711.20651-1-piliu@redhat.com> References: <20240313125711.20651-1-piliu@redhat.com> MIME-Version: 1.0 X-Scanned-By: MIMEDefang 3.4.1 on 10.11.54.5 X-Mimecast-Spam-Score: 0 X-Mimecast-Originator: redhat.com X-CRM114-Version: 20100106-BlameMichelson ( TRE 0.8.0 (BSD) ) MR-646709E3 X-CRM114-CacheID: sfid-20240313_060027_495610_AB028F58 X-CRM114-Status: GOOD ( 21.48 ) X-BeenThere: linux-arm-kernel@lists.infradead.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Sender: "linux-arm-kernel" Errors-To: linux-arm-kernel-bounces+linux-arm-kernel=archiver.kernel.org@lists.infradead.org The init_stack serves as stack for the C routines. For idmap, the mapping consist of five sections: kernel text section init_pg_dir, which needs to be accessed when create_kernel_mapping() __initdata, which contains data accessed by create_kernel_mapping() init_stack, which serves as the stack fdt Signed-off-by: Pingfan Liu Cc: Ard Biesheuvel Cc: Catalin Marinas Cc: Will Deacon Cc: Mark Rutland To: linux-arm-kernel@lists.infradead.org --- arch/arm64/include/asm/kernel-pgtable.h | 1 + arch/arm64/include/asm/mmu.h | 4 + arch/arm64/kernel/head.S | 171 +++++++++++++----------- arch/arm64/mm/mmu.c | 4 - 4 files changed, 96 insertions(+), 84 deletions(-) diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h index 85d26143faa5..796bf3d8c181 100644 --- a/arch/arm64/include/asm/kernel-pgtable.h +++ b/arch/arm64/include/asm/kernel-pgtable.h @@ -91,6 +91,7 @@ #else #define INIT_IDMAP_DIR_SIZE (INIT_IDMAP_DIR_PAGES * PAGE_SIZE) #endif +// #define INIT_IDMAP_DIR_PAGES EARLY_PAGES(KIMAGE_VADDR, _end + MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE, 1) /* Initial memory map size */ diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 2fcf51231d6e..b817b694d1ba 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -12,6 +12,10 @@ #define USER_ASID_FLAG (UL(1) << USER_ASID_BIT) #define TTBR_ASID_MASK (UL(0xffff) << 48) +#define NO_BLOCK_MAPPINGS BIT(0) +#define NO_CONT_MAPPINGS BIT(1) +#define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */ + #ifndef __ASSEMBLY__ #include diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 7b236994f0e1..e2fa6b95f809 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -332,79 +333,69 @@ SYM_FUNC_START_LOCAL(remap_region) SYM_FUNC_END(remap_region) SYM_FUNC_START_LOCAL(create_idmap) - mov x28, lr - /* - * The ID map carries a 1:1 mapping of the physical address range - * covered by the loaded image, which could be anywhere in DRAM. This - * means that the required size of the VA (== PA) space is decided at - * boot time, and could be more than the configured size of the VA - * space for ordinary kernel and user space mappings. - * - * There are three cases to consider here: - * - 39 <= VA_BITS < 48, and the ID map needs up to 48 VA bits to cover - * the placement of the image. In this case, we configure one extra - * level of translation on the fly for the ID map only. (This case - * also covers 42-bit VA/52-bit PA on 64k pages). - * - * - VA_BITS == 48, and the ID map needs more than 48 VA bits. This can - * only happen when using 64k pages, in which case we need to extend - * the root level table rather than add a level. Note that we can - * treat this case as 'always extended' as long as we take care not - * to program an unsupported T0SZ value into the TCR register. - * - * - Combinations that would require two additional levels of - * translation are not supported, e.g., VA_BITS==36 on 16k pages, or - * VA_BITS==39/4k pages with 5-level paging, where the input address - * requires more than 47 or 48 bits, respectively. - */ -#if (VA_BITS < 48) -#define IDMAP_PGD_ORDER (VA_BITS - PGDIR_SHIFT) -#define EXTRA_SHIFT (PGDIR_SHIFT + PAGE_SHIFT - 3) + adr_l x0, init_stack + add sp, x0, #THREAD_SIZE + sub sp, sp, #16 + stp lr, x0, [sp, #0] // x0 is useless, just to keep stack 16-bytes align - /* - * If VA_BITS < 48, we have to configure an additional table level. - * First, we have to verify our assumption that the current value of - * VA_BITS was chosen such that all translation levels are fully - * utilised, and that lowering T0SZ will always result in an additional - * translation level to be configured. - */ -#if VA_BITS != EXTRA_SHIFT -#error "Mismatch between VA_BITS and page size/number of translation levels" -#endif -#else -#define IDMAP_PGD_ORDER (PHYS_MASK_SHIFT - PGDIR_SHIFT) -#define EXTRA_SHIFT - /* - * If VA_BITS == 48, we don't have to configure an additional - * translation level, but the top-level table has more entries. - */ -#endif adrp x0, init_idmap_pg_dir - adrp x3, _text - adrp x6, _end + MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE - mov_q x7, SWAPPER_RX_MMUFLAGS - - map_memory x0, x1, x3, x6, x7, x3, IDMAP_PGD_ORDER, x10, x11, x12, x13, x14, EXTRA_SHIFT - - /* Remap the kernel page tables r/w in the ID map */ - adrp x1, _text - adrp x2, init_pg_dir - adrp x3, init_pg_end - bic x4, x2, #SWAPPER_BLOCK_SIZE - 1 - mov_q x5, SWAPPER_RW_MMUFLAGS - mov x6, #SWAPPER_BLOCK_SHIFT - bl remap_region - - /* Remap the FDT after the kernel image */ - adrp x1, _text - adrp x22, _end + SWAPPER_BLOCK_SIZE - bic x2, x22, #SWAPPER_BLOCK_SIZE - 1 + adrp x1, init_idmap_pg_end + sub x1, x1, x0 + bl headpool_init + mov x0, #0 + bl headpool_pgtable_alloc // return x0, containing init_idmap_pg_dir + mov x27, x0 // bake in case of flush + + adr_l x1, _text // phys + mov x2, x1 // virt for idmap + adr_l x3, _etext - 1 + sub x3, x3, x1 // size + ldr x4, =SWAPPER_RX_MMUFLAGS + adr_l x5, headpool_pgtable_alloc + mov x6, #0 + bl mmu_head_create_pgd_mapping + + mov x0, x27 // pgd + adr_l x1, init_pg_dir // phys + mov x2, x1 // virt for idmap + adr_l x3, init_pg_end + sub x3, x3, x1 + ldr x4, =SWAPPER_RW_MMUFLAGS + adr_l x5, headpool_pgtable_alloc + mov x6, #0 + bl mmu_head_create_pgd_mapping + + mov x0, x27 // pgd + adr_l x1, init_stack // kernel mapping need write-permission to use this stack + mov x2, x1 // virt for idmap + ldr x3, =THREAD_SIZE + ldr x4, =SWAPPER_RW_MMUFLAGS + adr_l x5, headpool_pgtable_alloc + mov x6, #0 + bl mmu_head_create_pgd_mapping + + mov x0, x27 // pgd + adr_l x1, __initdata_begin // kernel mapping need write-permission to it + mov x2, x1 // virt for idmap + adr_l x3, __initdata_end + sub x3, x3, x1 + ldr x4, =SWAPPER_RW_MMUFLAGS + adr_l x5, headpool_pgtable_alloc + mov x6, #0 + bl mmu_head_create_pgd_mapping + + + mov x0, x27 // pgd + mov x1, x21 // FDT phys + adr_l x2, _end + SWAPPER_BLOCK_SIZE // virt + mov x3, #(MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE) // size + ldr x4, =SWAPPER_RW_MMUFLAGS + adr_l x5, headpool_pgtable_alloc + mov x6, #0 + bl mmu_head_create_pgd_mapping + + adr_l x22, _end + SWAPPER_BLOCK_SIZE bfi x22, x21, #0, #SWAPPER_BLOCK_SHIFT // remapped FDT address - add x3, x2, #MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE - bic x4, x21, #SWAPPER_BLOCK_SIZE - 1 - mov_q x5, SWAPPER_RW_MMUFLAGS - mov x6, #SWAPPER_BLOCK_SHIFT - bl remap_region /* * Since the page tables have been populated with non-cacheable @@ -417,22 +408,42 @@ SYM_FUNC_START_LOCAL(create_idmap) adrp x0, init_idmap_pg_dir adrp x1, init_idmap_pg_end bl dcache_inval_poc -0: ret x28 + ldp lr, x0, [sp], #16 +0: ret SYM_FUNC_END(create_idmap) SYM_FUNC_START_LOCAL(create_kernel_mapping) + sub sp, sp, #80 + stp x0, x1, [sp, #0] + stp x2, x3, [sp, #16] + stp x4, x5, [sp, #32] + stp x6, x7, [sp, #48] + stp lr, xzr, [sp, #64] + adrp x0, init_pg_dir - mov_q x5, KIMAGE_VADDR // compile time __va(_text) + adrp x1, init_pg_end + sub x1, x1, x0 + bl headpool_init + mov x0, #0 + bl headpool_pgtable_alloc // return x0, containing init_pg_dir + + adrp x1, _text // runtime __pa(_text) + mov_q x2, KIMAGE_VADDR // compile time __va(_text) #ifdef CONFIG_RELOCATABLE - add x5, x5, x23 // add KASLR displacement + add x2, x2, x23 // add KASLR displacement #endif - adrp x6, _end // runtime __pa(_end) - adrp x3, _text // runtime __pa(_text) - sub x6, x6, x3 // _end - _text - add x6, x6, x5 // runtime __va(_end) - mov_q x7, SWAPPER_RW_MMUFLAGS - - map_memory x0, x1, x5, x6, x7, x3, (VA_BITS - PGDIR_SHIFT), x10, x11, x12, x13, x14 + adrp x3, _end // runtime __pa(_end) + sub x3, x3, x1 // _end - _text + ldr x4, =SWAPPER_RW_MMUFLAGS + adr_l x5, headpool_pgtable_alloc + mov x6, #0 + bl mmu_head_create_pgd_mapping + + ldp lr, xzr, [sp, #64] + ldp x6, x7, [sp, #48] + ldp x4, x5, [sp, #32] + ldp x2, x3, [sp, #16] + ldp x0, x1, [sp], #80 dsb ishst // sync with page table walker ret diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 80e49faaf066..e9748c7017dd 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -41,10 +41,6 @@ #include #include -#define NO_BLOCK_MAPPINGS BIT(0) -#define NO_CONT_MAPPINGS BIT(1) -#define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */ - int idmap_t0sz __ro_after_init; #if VA_BITS > 48