@@ -36,26 +36,33 @@ static DEFINE_SPINLOCK(cpupool_lock);
DEFINE_PER_CPU(struct cpupool *, cpupool);
+static void free_cpupool_struct(struct cpupool *c)
+{
+ if ( c )
+ {
+ free_cpumask_var(c->res_valid);
+ free_cpumask_var(c->cpu_valid);
+ }
+ xfree(c);
+}
+
static struct cpupool *alloc_cpupool_struct(void)
{
struct cpupool *c = xzalloc(struct cpupool);
- if ( !c || !zalloc_cpumask_var(&c->cpu_valid) )
+ if ( !c )
+ return NULL;
+
+ if ( !zalloc_cpumask_var(&c->cpu_valid) ||
+ !zalloc_cpumask_var(&c->res_valid) )
{
- xfree(c);
+ free_cpupool_struct(c);
c = NULL;
}
return c;
}
-static void free_cpupool_struct(struct cpupool *c)
-{
- if ( c )
- free_cpumask_var(c->cpu_valid);
- xfree(c);
-}
-
/*
* find a cpupool by it's id. to be called with cpupool lock held
* if exact is not specified, the first cpupool with an id larger or equal to
@@ -269,6 +276,7 @@ static int cpupool_assign_cpu_locked(struct cpupool *c, unsigned int cpu)
cpupool_cpu_moving = NULL;
}
cpumask_set_cpu(cpu, c->cpu_valid);
+ cpumask_and(c->res_valid, c->cpu_valid, sched_res_mask);
rcu_read_lock(&domlist_read_lock);
for_each_domain_in_cpupool(d, c)
@@ -361,6 +369,7 @@ static int cpupool_unassign_cpu_start(struct cpupool *c, unsigned int cpu)
atomic_inc(&c->refcnt);
cpupool_cpu_moving = c;
cpumask_clear_cpu(cpu, c->cpu_valid);
+ cpumask_and(c->res_valid, c->cpu_valid, sched_res_mask);
out:
spin_unlock(&cpupool_lock);
@@ -584,7 +584,7 @@ void domain_update_node_affinity(struct domain *d)
return;
}
- online = cpupool_domain_cpumask(d);
+ online = cpupool_domain_master_cpumask(d);
spin_lock(&d->node_affinity_lock);
@@ -619,7 +619,7 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)
if ( op->cmd == XEN_DOMCTL_setvcpuaffinity )
{
cpumask_var_t new_affinity, old_affinity;
- cpumask_t *online = cpupool_domain_cpumask(v->domain);
+ cpumask_t *online = cpupool_domain_master_cpumask(v->domain);
/*
* We want to be able to restore hard affinity if we are trying
@@ -614,7 +614,7 @@ a653sched_pick_resource(const struct scheduler *ops,
* If present, prefer unit's current processor, else
* just find the first valid unit.
*/
- online = cpupool_domain_cpumask(unit->domain);
+ online = cpupool_domain_master_cpumask(unit->domain);
cpu = cpumask_first(online);
@@ -361,7 +361,7 @@ static inline void __runq_tickle(struct csched_unit *new)
ASSERT(cur);
cpumask_clear(&mask);
- online = cpupool_domain_cpumask(new->sdom->dom);
+ online = cpupool_domain_master_cpumask(new->sdom->dom);
cpumask_and(&idle_mask, prv->idlers, online);
idlers_empty = cpumask_empty(&idle_mask);
@@ -724,7 +724,7 @@ _csched_cpu_pick(const struct scheduler *ops, const struct sched_unit *unit,
/* We must always use cpu's scratch space */
cpumask_t *cpus = cpumask_scratch_cpu(cpu);
cpumask_t idlers;
- cpumask_t *online = cpupool_domain_cpumask(unit->domain);
+ cpumask_t *online = cpupool_domain_master_cpumask(unit->domain);
struct csched_pcpu *spc = NULL;
int balance_step;
@@ -705,7 +705,7 @@ static int get_fallback_cpu(struct csched2_unit *svc)
affinity_balance_cpumask(unit, bs, cpumask_scratch_cpu(cpu));
cpumask_and(cpumask_scratch_cpu(cpu), cpumask_scratch_cpu(cpu),
- cpupool_domain_cpumask(unit->domain));
+ cpupool_domain_master_cpumask(unit->domain));
/*
* This is cases 1 or 3 (depending on bs): if processor is (still)
@@ -1440,7 +1440,7 @@ runq_tickle(const struct scheduler *ops, struct csched2_unit *new, s_time_t now)
struct sched_unit *unit = new->unit;
unsigned int bs, cpu = sched_unit_master(unit);
struct csched2_runqueue_data *rqd = c2rqd(ops, cpu);
- cpumask_t *online = cpupool_domain_cpumask(unit->domain);
+ cpumask_t *online = cpupool_domain_master_cpumask(unit->domain);
cpumask_t mask;
ASSERT(new->rqd == rqd);
@@ -2243,7 +2243,7 @@ csched2_res_pick(const struct scheduler *ops, const struct sched_unit *unit)
}
cpumask_and(cpumask_scratch_cpu(cpu), unit->cpu_hard_affinity,
- cpupool_domain_cpumask(unit->domain));
+ cpupool_domain_master_cpumask(unit->domain));
/*
* First check to see if we're here because someone else suggested a place
@@ -2358,8 +2358,8 @@ csched2_res_pick(const struct scheduler *ops, const struct sched_unit *unit)
* ok because:
* - we know that unit->cpu_hard_affinity and ->cpu_soft_affinity have
* a non-empty intersection (because has_soft is true);
- * - we have unit->cpu_hard_affinity & cpupool_domain_cpumask() already
- * in cpumask_scratch, we do save a lot doing like this.
+ * - we have unit->cpu_hard_affinity & cpupool_domain_master_cpumask()
+ * already in cpumask_scratch, we do save a lot doing like this.
*
* It's kind of like open coding affinity_balance_cpumask() but, in
* this specific case, calling that would mean a lot of (unnecessary)
@@ -2378,7 +2378,7 @@ csched2_res_pick(const struct scheduler *ops, const struct sched_unit *unit)
* affinity, so go for it.
*
* cpumask_scratch already has unit->cpu_hard_affinity &
- * cpupool_domain_cpumask() in it, so it's enough that we filter
+ * cpupool_domain_master_cpumask() in it, so it's enough that we filter
* with the cpus of the runq.
*/
cpumask_and(cpumask_scratch_cpu(cpu), cpumask_scratch_cpu(cpu),
@@ -2513,7 +2513,7 @@ static void migrate(const struct scheduler *ops,
_runq_deassign(svc);
cpumask_and(cpumask_scratch_cpu(cpu), unit->cpu_hard_affinity,
- cpupool_domain_cpumask(unit->domain));
+ cpupool_domain_master_cpumask(unit->domain));
cpumask_and(cpumask_scratch_cpu(cpu), cpumask_scratch_cpu(cpu),
&trqd->active);
sched_set_res(unit,
@@ -2547,7 +2547,7 @@ static bool unit_is_migrateable(struct csched2_unit *svc,
int cpu = sched_unit_master(unit);
cpumask_and(cpumask_scratch_cpu(cpu), unit->cpu_hard_affinity,
- cpupool_domain_cpumask(unit->domain));
+ cpupool_domain_master_cpumask(unit->domain));
return !(svc->flags & CSFLAG_runq_migrate_request) &&
cpumask_intersects(cpumask_scratch_cpu(cpu), &rqd->active);
@@ -2763,7 +2763,7 @@ csched2_unit_migrate(
* v->processor will be chosen, and during actual domain unpause that
* the unit will be assigned to and added to the proper runqueue.
*/
- if ( unlikely(!cpumask_test_cpu(new_cpu, cpupool_domain_cpumask(d))) )
+ if ( unlikely(!cpumask_test_cpu(new_cpu, cpupool_domain_master_cpumask(d))) )
{
ASSERT(system_state == SYS_STATE_suspend);
if ( unit_on_runq(svc) )
@@ -3069,7 +3069,7 @@ csched2_alloc_domdata(const struct scheduler *ops, struct domain *dom)
sdom->nr_units = 0;
init_timer(&sdom->repl_timer, replenish_domain_budget, sdom,
- cpumask_any(cpupool_domain_cpumask(dom)));
+ cpumask_any(cpupool_domain_master_cpumask(dom)));
spin_lock_init(&sdom->budget_lock);
INIT_LIST_HEAD(&sdom->parked_units);
@@ -3317,7 +3317,7 @@ runq_candidate(struct csched2_runqueue_data *rqd,
cpumask_scratch);
if ( unlikely(!cpumask_test_cpu(cpu, cpumask_scratch)) )
{
- cpumask_t *online = cpupool_domain_cpumask(scurr->unit->domain);
+ cpumask_t *online = cpupool_domain_master_cpumask(scurr->unit->domain);
/* Ok, is any of the pcpus in scurr soft-affinity idle? */
cpumask_and(cpumask_scratch, cpumask_scratch, &rqd->idle);
@@ -125,7 +125,7 @@ static inline bool unit_check_affinity(struct sched_unit *unit,
{
affinity_balance_cpumask(unit, balance_step, cpumask_scratch_cpu(cpu));
cpumask_and(cpumask_scratch_cpu(cpu), cpumask_scratch_cpu(cpu),
- cpupool_domain_cpumask(unit->domain));
+ cpupool_domain_master_cpumask(unit->domain));
return cpumask_test_cpu(cpu, cpumask_scratch_cpu(cpu));
}
@@ -266,7 +266,7 @@ pick_res(struct null_private *prv, const struct sched_unit *unit)
{
unsigned int bs;
unsigned int cpu = sched_unit_master(unit), new_cpu;
- cpumask_t *cpus = cpupool_domain_cpumask(unit->domain);
+ cpumask_t *cpus = cpupool_domain_master_cpumask(unit->domain);
ASSERT(spin_is_locked(get_sched_res(cpu)->schedule_lock));
@@ -467,7 +467,7 @@ static void null_unit_insert(const struct scheduler *ops,
lock = unit_schedule_lock(unit);
cpumask_and(cpumask_scratch_cpu(cpu), unit->cpu_hard_affinity,
- cpupool_domain_cpumask(unit->domain));
+ cpupool_domain_master_cpumask(unit->domain));
/* If the pCPU is free, we assign unit to it */
if ( likely(per_cpu(npc, cpu).unit == NULL) )
@@ -579,7 +579,7 @@ static void null_unit_wake(const struct scheduler *ops,
spin_unlock(&prv->waitq_lock);
cpumask_and(cpumask_scratch_cpu(cpu), unit->cpu_hard_affinity,
- cpupool_domain_cpumask(unit->domain));
+ cpupool_domain_master_cpumask(unit->domain));
if ( !cpumask_intersects(&prv->cpus_free, cpumask_scratch_cpu(cpu)) )
{
@@ -326,7 +326,7 @@ rt_dump_unit(const struct scheduler *ops, const struct rt_unit *svc)
*/
mask = cpumask_scratch_cpu(sched_unit_master(svc->unit));
- cpupool_mask = cpupool_domain_cpumask(svc->unit->domain);
+ cpupool_mask = cpupool_domain_master_cpumask(svc->unit->domain);
cpumask_and(mask, cpupool_mask, svc->unit->cpu_hard_affinity);
printk("[%5d.%-2u] cpu %u, (%"PRI_stime", %"PRI_stime"),"
" cur_b=%"PRI_stime" cur_d=%"PRI_stime" last_start=%"PRI_stime"\n"
@@ -642,7 +642,7 @@ rt_res_pick(const struct scheduler *ops, const struct sched_unit *unit)
cpumask_t *online;
int cpu;
- online = cpupool_domain_cpumask(unit->domain);
+ online = cpupool_domain_master_cpumask(unit->domain);
cpumask_and(&cpus, online, unit->cpu_hard_affinity);
cpu = cpumask_test_cpu(sched_unit_master(unit), &cpus)
@@ -1016,7 +1016,7 @@ runq_pick(const struct scheduler *ops, const cpumask_t *mask)
iter_svc = q_elem(iter);
/* mask cpu_hard_affinity & cpupool & mask */
- online = cpupool_domain_cpumask(iter_svc->unit->domain);
+ online = cpupool_domain_master_cpumask(iter_svc->unit->domain);
cpumask_and(&cpu_common, online, iter_svc->unit->cpu_hard_affinity);
cpumask_and(&cpu_common, mask, &cpu_common);
if ( cpumask_empty(&cpu_common) )
@@ -1191,7 +1191,7 @@ runq_tickle(const struct scheduler *ops, struct rt_unit *new)
if ( new == NULL || is_idle_unit(new->unit) )
return;
- online = cpupool_domain_cpumask(new->unit->domain);
+ online = cpupool_domain_master_cpumask(new->unit->domain);
cpumask_and(¬_tickled, online, new->unit->cpu_hard_affinity);
cpumask_andnot(¬_tickled, ¬_tickled, &prv->tickled);
@@ -63,6 +63,7 @@ integer_param("sched_ratelimit_us", sched_ratelimit_us);
/* Number of vcpus per struct sched_unit. */
static unsigned int __read_mostly sched_granularity = 1;
+const cpumask_t *sched_res_mask = &cpumask_all;
/* Common lock for free cpus. */
static DEFINE_SPINLOCK(sched_free_cpu_lock);
@@ -188,7 +189,7 @@ static inline struct scheduler *vcpu_scheduler(const struct vcpu *v)
{
return unit_scheduler(v->sched_unit);
}
-#define VCPU2ONLINE(_v) cpupool_domain_cpumask((_v)->domain)
+#define VCPU2ONLINE(_v) cpupool_domain_master_cpumask((_v)->domain)
static inline void trace_runstate_change(struct vcpu *v, int new_state)
{
@@ -425,9 +426,9 @@ static unsigned int sched_select_initial_cpu(const struct vcpu *v)
cpumask_clear(cpus);
for_each_node_mask ( node, d->node_affinity )
cpumask_or(cpus, cpus, &node_to_cpumask(node));
- cpumask_and(cpus, cpus, cpupool_domain_cpumask(d));
+ cpumask_and(cpus, cpus, d->cpupool->cpu_valid);
if ( cpumask_empty(cpus) )
- cpumask_copy(cpus, cpupool_domain_cpumask(d));
+ cpumask_copy(cpus, d->cpupool->cpu_valid);
if ( v->vcpu_id == 0 )
cpu_ret = cpumask_first(cpus);
@@ -973,7 +974,7 @@ void restore_vcpu_affinity(struct domain *d)
lock = unit_schedule_lock_irq(unit);
cpumask_and(cpumask_scratch_cpu(cpu), unit->cpu_hard_affinity,
- cpupool_domain_cpumask(d));
+ cpupool_domain_master_cpumask(d));
if ( cpumask_empty(cpumask_scratch_cpu(cpu)) )
{
if ( sched_check_affinity_broken(unit) )
@@ -981,7 +982,7 @@ void restore_vcpu_affinity(struct domain *d)
sched_set_affinity(unit, unit->cpu_hard_affinity_saved, NULL);
sched_reset_affinity_broken(unit);
cpumask_and(cpumask_scratch_cpu(cpu), unit->cpu_hard_affinity,
- cpupool_domain_cpumask(d));
+ cpupool_domain_master_cpumask(d));
}
if ( cpumask_empty(cpumask_scratch_cpu(cpu)) )
@@ -991,7 +992,7 @@ void restore_vcpu_affinity(struct domain *d)
unit->vcpu_list);
sched_set_affinity(unit, &cpumask_all, NULL);
cpumask_and(cpumask_scratch_cpu(cpu), unit->cpu_hard_affinity,
- cpupool_domain_cpumask(d));
+ cpupool_domain_master_cpumask(d));
}
}
@@ -22,6 +22,8 @@ extern cpumask_t cpupool_free_cpus;
#define SCHED_DEFAULT_RATELIMIT_US 1000
extern int sched_ratelimit_us;
+/* Scheduling resource mask. */
+extern const cpumask_t *sched_res_mask;
/*
* In order to allow a scheduler to remap the lock->cpu mapping,
@@ -535,6 +537,7 @@ struct cpupool
int cpupool_id;
unsigned int n_dom;
cpumask_var_t cpu_valid; /* all cpus assigned to pool */
+ cpumask_var_t res_valid; /* all scheduling resources of pool */
struct cpupool *next;
struct scheduler *sched;
atomic_t refcnt;
@@ -543,14 +546,14 @@ struct cpupool
#define cpupool_online_cpumask(_pool) \
(((_pool) == NULL) ? &cpu_online_map : (_pool)->cpu_valid)
-static inline cpumask_t *cpupool_domain_cpumask(const struct domain *d)
+static inline cpumask_t *cpupool_domain_master_cpumask(const struct domain *d)
{
/*
* d->cpupool is NULL only for the idle domain, and no one should
* be interested in calling this for the idle domain.
*/
ASSERT(d->cpupool != NULL);
- return d->cpupool->cpu_valid;
+ return d->cpupool->res_valid;
}
/*
@@ -590,7 +593,7 @@ static inline cpumask_t *cpupool_domain_cpumask(const struct domain *d)
static inline int has_soft_affinity(const struct sched_unit *unit)
{
return unit->soft_aff_effective &&
- !cpumask_subset(cpupool_domain_cpumask(unit->domain),
+ !cpumask_subset(cpupool_domain_master_cpumask(unit->domain),
unit->cpu_soft_affinity);
}