@@ -4584,6 +4584,104 @@ static unsigned long __move_all_objects_to(struct kmem_cache *s, int node)
return left;
}
+
+/*
+ * __move_n_slabs() - Attempt to move 'num' slabs to target_node,
+ * Return: The number of slabs moved or error code.
+ */
+static long __move_n_slabs(struct kmem_cache *s, int node, int target_node,
+ long num)
+{
+ struct kmem_cache_node *n = get_node(s, node);
+ LIST_HEAD(move_list);
+ struct page *page, *page2;
+ unsigned long flags;
+ void **scratch;
+ long done = 0;
+
+ if (node == target_node)
+ return -EINVAL;
+
+ scratch = alloc_scratch(s);
+ if (!scratch)
+ return -ENOMEM;
+
+ spin_lock_irqsave(&n->list_lock, flags);
+ list_for_each_entry_safe(page, page2, &n->full, lru) {
+ if (!slab_trylock(page))
+ /* Busy slab. Get out of the way */
+ continue;
+
+ list_move(&page->lru, &move_list);
+ page->frozen = 1;
+ slab_unlock(page);
+
+ if (++done >= num)
+ break;
+ }
+ spin_unlock_irqrestore(&n->list_lock, flags);
+
+ list_for_each_entry(page, &move_list, lru) {
+ if (page->inuse)
+ __move(page, scratch, target_node);
+ }
+ kfree(scratch);
+
+ /* Inspect results and dispose of pages */
+ spin_lock_irqsave(&n->list_lock, flags);
+ list_for_each_entry_safe(page, page2, &move_list, lru) {
+ list_del(&page->lru);
+ slab_lock(page);
+ page->frozen = 0;
+
+ if (page->inuse) {
+ /*
+ * This is best effort only, if slab still has
+ * objects just put it back on the partial list.
+ */
+ n->nr_partial++;
+ list_add_tail(&page->lru, &n->partial);
+ slab_unlock(page);
+ } else {
+ slab_unlock(page);
+ discard_slab(s, page);
+ }
+ }
+ spin_unlock_irqrestore(&n->list_lock, flags);
+
+ return done;
+}
+
+/*
+ * __balance_nodes_partial() - Balance partial objects.
+ * @s: The cache we are working on.
+ *
+ * Attempt to balance the objects that are in partial slabs evenly
+ * across all nodes.
+ */
+static void __balance_nodes_partial(struct kmem_cache *s)
+{
+ struct kmem_cache_node *n = get_node(s, 0);
+ unsigned long desired_nr_slabs_per_node;
+ unsigned long nr_slabs;
+ int nr_nodes = 0;
+ int nid;
+
+ (void)__move_all_objects_to(s, 0);
+
+ for_each_node_state(nid, N_NORMAL_MEMORY)
+ nr_nodes++;
+
+ nr_slabs = atomic_long_read(&n->nr_slabs);
+ desired_nr_slabs_per_node = nr_slabs / nr_nodes;
+
+ for_each_node_state(nid, N_NORMAL_MEMORY) {
+ if (nid == 0)
+ continue;
+
+ __move_n_slabs(s, 0, nid, desired_nr_slabs_per_node);
+ }
+}
#endif
/**
@@ -5836,6 +5934,22 @@ static ssize_t move_store(struct kmem_cache *s, const char *buf, size_t length)
return length;
}
SLAB_ATTR(move);
+
+static ssize_t balance_show(struct kmem_cache *s, char *buf)
+{
+ return 0;
+}
+
+static ssize_t balance_store(struct kmem_cache *s,
+ const char *buf, size_t length)
+{
+ if (buf[0] == '1')
+ __balance_nodes_partial(s);
+ else
+ return -EINVAL;
+ return length;
+}
+SLAB_ATTR(balance);
#endif /* CONFIG_SMO_NODE */
#ifdef CONFIG_NUMA
@@ -5964,6 +6078,7 @@ static struct attribute *slab_attrs[] = {
&shrink_attr.attr,
#ifdef CONFIG_SMO_NODE
&move_attr.attr,
+ &balance_attr.attr,
#endif
&slabs_cpu_partial_attr.attr,
#ifdef CONFIG_SLUB_DEBUG
We have just implemented Slab Movable Objects (SMO). On NUMA systems slabs can become unbalanced i.e. many objects on one node while other nodes have few objects. Using SMO we can balance the objects across all the nodes. The algorithm used is as follows: 1. Move all objects to node 0 (this has the effect of defragmenting the cache). 2. Calculate the desired number of slabs for each node (this is done using the approximation nr_slabs / nr_nodes). 3. Loop over the nodes moving the desired number of slabs from node 0 to the node. Feature is conditionally built in with CONFIG_SMO_NODE, this is because we need the full list (we enable SLUB_DEBUG to get this). Future version may separate final list out of SLUB_DEBUG. Expose this functionality to userspace via a sysfs entry. Add sysfs entry: /sysfs/kernel/slab/<cache>/balance Write of '1' to this file triggers balance, no other value accepted. This feature relies on SMO being enable for the cache, this is done with a call to, after the isolate/migrate functions have been defined. kmem_cache_setup_mobility(s, isolate, migrate) Signed-off-by: Tobin C. Harding <tobin@kernel.org> --- mm/slub.c | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+)