@@ -7,6 +7,8 @@
#include <asm-generic/memory_metadata.h>
+#include <asm/mte.h>
+
#ifdef CONFIG_MEMORY_METADATA
static inline bool metadata_storage_enabled(void)
{
@@ -16,6 +18,9 @@ static inline bool alloc_can_use_metadata_pages(gfp_t gfp_mask)
{
return false;
}
+
+#define page_has_metadata(page) page_mte_tagged(page)
+
#endif /* CONFIG_MEMORY_METADATA */
#endif /* __ASM_MEMORY_METADATA_H */
@@ -3,6 +3,7 @@
#define __ASM_GENERIC_MEMORY_METADATA_H
#include <linux/gfp.h>
+#include <linux/mm_types.h>
extern unsigned long totalmetadata_pages;
@@ -15,6 +16,10 @@ static inline bool alloc_can_use_metadata_pages(gfp_t gfp_mask)
{
return false;
}
+static inline bool page_has_metadata(struct page *page)
+{
+ return false;
+}
#endif /* !CONFIG_MEMORY_METADATA */
#endif /* __ASM_GENERIC_MEMORY_METADATA_H */
@@ -91,7 +91,7 @@ extern const char * const migratetype_names[MIGRATE_TYPES];
static inline bool is_migrate_movable(int mt)
{
- return is_migrate_cma(mt) || mt == MIGRATE_MOVABLE;
+ return is_migrate_cma(mt) || is_migrate_metadata(mt) || mt == MIGRATE_MOVABLE;
}
/*
@@ -1153,6 +1153,9 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
nr_isolated += folio_nr_pages(folio);
nr_scanned += folio_nr_pages(folio) - 1;
+ if (page_has_metadata(&folio->page))
+ cc->source_has_metadata = true;
+
/*
* Avoid isolating too much unless this block is being
* fully scanned (e.g. dirty/writeback pages, parallel allocation)
@@ -1328,6 +1331,15 @@ static bool suitable_migration_source(struct compact_control *cc,
static bool suitable_migration_target(struct compact_control *cc,
struct page *page)
{
+ int block_mt;
+
+ block_mt = get_pageblock_migratetype(page);
+
+ /* Pages from MIGRATE_METADATA cannot have metadata. */
+ if (is_migrate_metadata(block_mt) && cc->source_has_metadata)
+ return false;
+
+
/* If the page is a large free page, then disallow migration */
if (PageBuddy(page)) {
/*
@@ -1342,8 +1354,11 @@ static bool suitable_migration_target(struct compact_control *cc,
if (cc->ignore_block_suitable)
return true;
- /* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */
- if (is_migrate_movable(get_pageblock_migratetype(page)))
+ /*
+ * If the block is MIGRATE_MOVABLE, MIGRATE_CMA or MIGRATE_METADATA,
+ * allow migration.
+ */
+ if (is_migrate_movable(block_mt))
return true;
/* Otherwise skip the block */
@@ -491,6 +491,7 @@ struct compact_control {
* ensure forward progress.
*/
bool alloc_contig; /* alloc_contig_range allocation */
+ bool source_has_metadata; /* source pages have associated metadata */
};
/*
MIGRATE_METADATA pages are special because for the one architecture (arm64) that use them, it is not possible to have metadata associated with a page used to store metadata. To avoid a situation where a page with metadata is being migrated to a page which cannot have metadata, keep track of whether such pages have been isolated as the source for migration. When allocating a destination page for migration, deny allocations from MIGRATE_METADATA if that's the case. fast_isolate_freepages() takes pages only from the MIGRATE_MOVABLE list, which means it is not necessary to have a similar check, as MIGRATE_METADATA pages will never be considered. Signed-off-by: Alexandru Elisei <alexandru.elisei@arm.com> --- arch/arm64/include/asm/memory_metadata.h | 5 +++++ include/asm-generic/memory_metadata.h | 5 +++++ include/linux/mmzone.h | 2 +- mm/compaction.c | 19 +++++++++++++++++-- mm/internal.h | 1 + 5 files changed, 29 insertions(+), 3 deletions(-)