@@ -225,6 +225,7 @@ struct mem_cgroup {
/* Legacy consumer-oriented counters */
struct page_counter kmem; /* v1 only */
struct page_counter tcpmem; /* v1 only */
+ struct page_counter toptier;
/* Range enforcement for interrupt charges */
struct work_struct high_work;
@@ -253,6 +253,13 @@ struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
}
+static inline bool top_tier(struct page *page)
+{
+ int nid = page_to_nid(page);
+
+ return node_state(nid, N_TOPTIER);
+}
+
#ifdef CONFIG_MEMCG_KMEM
extern spinlock_t css_set_lock;
@@ -951,6 +958,23 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
__this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
}
+static inline void mem_cgroup_charge_toptier(struct mem_cgroup *memcg,
+ struct page *page,
+ int nr_pages)
+{
+ if (!top_tier(page))
+ return;
+
+ if (nr_pages >= 0)
+ page_counter_charge(&memcg->toptier,
+ (unsigned long) nr_pages);
+ else {
+ nr_pages = -nr_pages;
+ page_counter_uncharge(&memcg->toptier,
+ (unsigned long) nr_pages);
+ }
+}
+
static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
enum mem_cgroup_events_target target)
{
@@ -2932,6 +2956,7 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg)
* - exclusive reference
*/
page->memcg_data = (unsigned long)memcg;
+ mem_cgroup_charge_toptier(memcg, page, thp_nr_pages(page));
}
#ifdef CONFIG_MEMCG_KMEM
@@ -3138,6 +3163,7 @@ int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
if (!ret) {
page->memcg_data = (unsigned long)memcg |
MEMCG_DATA_KMEM;
+ mem_cgroup_charge_toptier(memcg, page, 1 << order);
return 0;
}
css_put(&memcg->css);
@@ -3161,6 +3187,7 @@ void __memcg_kmem_uncharge_page(struct page *page, int order)
VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
__memcg_kmem_uncharge(memcg, nr_pages);
page->memcg_data = 0;
+ mem_cgroup_charge_toptier(memcg, page, -nr_pages);
css_put(&memcg->css);
}
@@ -5389,11 +5416,13 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
page_counter_init(&memcg->memory, &parent->memory);
page_counter_init(&memcg->swap, &parent->swap);
+ page_counter_init(&memcg->toptier, &parent->toptier);
page_counter_init(&memcg->kmem, &parent->kmem);
page_counter_init(&memcg->tcpmem, &parent->tcpmem);
} else {
page_counter_init(&memcg->memory, NULL);
page_counter_init(&memcg->swap, NULL);
+ page_counter_init(&memcg->toptier, NULL);
page_counter_init(&memcg->kmem, NULL);
page_counter_init(&memcg->tcpmem, NULL);
@@ -5745,6 +5774,8 @@ static int mem_cgroup_move_account(struct page *page,
css_put(&from->css);
page->memcg_data = (unsigned long)to;
+ mem_cgroup_charge_toptier(to, page, nr_pages);
+ mem_cgroup_charge_toptier(from, page, -nr_pages);
__unlock_page_memcg(from);
@@ -6832,6 +6863,7 @@ struct uncharge_gather {
unsigned long nr_pages;
unsigned long pgpgout;
unsigned long nr_kmem;
+ unsigned long nr_toptier;
struct page *dummy_page;
};
@@ -6846,6 +6878,7 @@ static void uncharge_batch(const struct uncharge_gather *ug)
if (!mem_cgroup_is_root(ug->memcg)) {
page_counter_uncharge(&ug->memcg->memory, ug->nr_pages);
+ page_counter_uncharge(&ug->memcg->toptier, ug->nr_toptier);
if (do_memsw_account())
page_counter_uncharge(&ug->memcg->memsw, ug->nr_pages);
if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
@@ -6891,6 +6924,8 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug)
nr_pages = compound_nr(page);
ug->nr_pages += nr_pages;
+ if (top_tier(page))
+ ug->nr_toptier += nr_pages;
if (PageMemcgKmem(page))
ug->nr_kmem += nr_pages;
@@ -7216,8 +7251,10 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
page->memcg_data = 0;
- if (!mem_cgroup_is_root(memcg))
+ if (!mem_cgroup_is_root(memcg)) {
page_counter_uncharge(&memcg->memory, nr_entries);
+ mem_cgroup_charge_toptier(memcg, page, -nr_entries);
+ }
if (!cgroup_memory_noswap && memcg != swap_memcg) {
if (!mem_cgroup_is_root(swap_memcg))
For each memory cgroup, account its usage of the top tier memory at the time a top tier page is assigned and uncharged from the cgroup. Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com> --- include/linux/memcontrol.h | 1 + mm/memcontrol.c | 39 +++++++++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-)