@@ -10,6 +10,7 @@
#include <xen/multicall.h>
#include <xen/guest_access.h>
#include <xen/perfc.h>
+#include <xen/preempt.h>
#include <xen/trace.h>
#include <asm/current.h>
#include <asm/hardirq.h>
@@ -21,13 +21,15 @@
#include <xen/preempt.h>
#include <xen/irq.h>
+#include <xen/rcupdate.h>
#include <asm/system.h>
DEFINE_PER_CPU(unsigned int, __preempt_count);
bool_t in_atomic(void)
{
- return preempt_count() || in_irq() || !local_irq_is_enabled();
+ return preempt_count() || in_irq() || !local_irq_is_enabled() ||
+ !rcu_quiesce_allowed();
}
#ifndef NDEBUG
@@ -36,5 +38,6 @@ void ASSERT_NOT_IN_ATOMIC(void)
ASSERT(!preempt_count());
ASSERT(!in_irq());
ASSERT(local_irq_is_enabled());
+ ASSERT(rcu_quiesce_allowed());
}
#endif
@@ -46,6 +46,10 @@
#include <xen/cpu.h>
#include <xen/stop_machine.h>
+#ifndef NDEBUG
+DEFINE_PER_CPU(unsigned int, rcu_lock_cnt);
+#endif
+
/* Global control variables for rcupdate callback mechanism. */
static struct rcu_ctrlblk {
long cur; /* Current batch number. */
@@ -30,6 +30,8 @@ static void __do_softirq(unsigned long ignore_mask, bool rcu_allowed)
unsigned int i, cpu;
unsigned long pending;
+ ASSERT(!rcu_allowed || rcu_quiesce_allowed());
+
for ( ; ; )
{
/*
@@ -19,6 +19,7 @@
* along with this program; If not, see <http://www.gnu.org/licenses/>.
*/
+#include <xen/preempt.h>
#include <xen/sched.h>
#include <xen/softirq.h>
#include <xen/wait.h>
@@ -32,12 +32,42 @@
#define __XEN_RCUPDATE_H
#include <xen/cache.h>
+#include <xen/compiler.h>
#include <xen/spinlock.h>
#include <xen/cpumask.h>
-#include <xen/preempt.h>
+#include <xen/percpu.h>
#define __rcu
+#ifndef NDEBUG
+DECLARE_PER_CPU(unsigned int, rcu_lock_cnt);
+
+static inline void rcu_quiesce_disable(void)
+{
+ this_cpu(rcu_lock_cnt)++;
+ barrier();
+}
+
+static inline void rcu_quiesce_enable(void)
+{
+ barrier();
+ this_cpu(rcu_lock_cnt)--;
+}
+
+static inline bool rcu_quiesce_allowed(void)
+{
+ return !this_cpu(rcu_lock_cnt);
+}
+
+#else
+static inline void rcu_quiesce_disable(void) { }
+static inline void rcu_quiesce_enable(void) { }
+static inline bool rcu_quiesce_allowed(void)
+{
+ return true;
+}
+#endif
+
/**
* struct rcu_head - callback structure for use with RCU
* @next: next update requests in a list
@@ -91,16 +121,23 @@ typedef struct _rcu_read_lock rcu_read_lock_t;
* will be deferred until the outermost RCU read-side critical section
* completes.
*
- * It is illegal to block while in an RCU read-side critical section.
+ * It is illegal to process softirqs while in an RCU read-side critical section.
*/
-#define rcu_read_lock(x) ({ ((void)(x)); preempt_disable(); })
+static inline void rcu_read_lock(rcu_read_lock_t *lock)
+{
+ rcu_quiesce_disable();
+}
/**
* rcu_read_unlock - marks the end of an RCU read-side critical section.
*
* See rcu_read_lock() for more information.
*/
-#define rcu_read_unlock(x) ({ ((void)(x)); preempt_enable(); })
+static inline void rcu_read_unlock(rcu_read_lock_t *lock)
+{
+ ASSERT(!rcu_quiesce_allowed());
+ rcu_quiesce_enable();
+}
/*
* So where is rcu_write_lock()? It does not exist, as there is no
Xen's RCU implementation relies on no softirq handling taking place while being in a RCU critical section. Add ASSERT()s in debug builds in order to catch any violations. For that purpose modify rcu_read_[un]lock() to use a dedicated percpu counter instead of preempt_[en|dis]able() as this enables to test that condition in __do_softirq() (ASSERT_NOT_IN_ATOMIC() is not usable there due to __cpu_up() calling process_pending_softirqs() while holding the cpu hotplug lock). Dropping the now no longer needed #include of preempt.h in rcupdate.h requires adding it in some sources. While at it switch the rcu_read_[un]lock() implementation to static inline functions instead of macros. Signed-off-by: Juergen Gross <jgross@suse.com> --- V3: - add barriers to rcu_[en|dis]able() (Roger Pau Monné) - add rcu_quiesce_allowed() to ASSERT_NOT_IN_ATOMIC (Roger Pau Monné) - convert macros to static inline functions - add sanity check in rcu_read_unlock() V4: - use barrier() in rcu_[en|dis]able() (Julien Grall) --- xen/common/multicall.c | 1 + xen/common/preempt.c | 5 ++++- xen/common/rcupdate.c | 4 ++++ xen/common/softirq.c | 2 ++ xen/common/wait.c | 1 + xen/include/xen/rcupdate.h | 45 +++++++++++++++++++++++++++++++++++++++++---- 6 files changed, 53 insertions(+), 5 deletions(-)