diff --git a/xen/arch/x86/microcode.c b/xen/arch/x86/microcode.c
index cbaf13d..67549be 100644
--- a/xen/arch/x86/microcode.c
+++ b/xen/arch/x86/microcode.c
@@ -30,18 +30,41 @@
 #include <xen/smp.h>
 #include <xen/softirq.h>
 #include <xen/spinlock.h>
+#include <xen/stop_machine.h>
 #include <xen/tasklet.h>
 #include <xen/guest_access.h>
 #include <xen/earlycpio.h>
+#include <xen/watchdog.h>
 
+#include <asm/delay.h>
 #include <asm/msr.h>
 #include <asm/processor.h>
 #include <asm/setup.h>
 #include <asm/microcode.h>
 
+/*
+ * Before performing a late microcode update on any thread, we
+ * rendezvous all cpus in stop_machine context. The timeout for
+ * waiting for cpu rendezvous is 30ms. It is the timeout used by
+ * live patching
+ */
+#define MICROCODE_CALLIN_TIMEOUT_US 30000
+
+/*
+ * Timeout for each thread to complete update is set to 1s. It is a
+ * conservative choice considering all possible interference.
+ */
+#define MICROCODE_UPDATE_TIMEOUT_US 1000000
+
 static module_t __initdata ucode_mod;
 static signed int __initdata ucode_mod_idx;
 static bool_t __initdata ucode_mod_forced;
+static unsigned int nr_cores;
+static enum {
+    LOADING_EXITED,
+    LOADING_ENTERED,
+    LOADING_ABORTED,
+} loading_state;
 
 /*
  * If we scan the initramfs.cpio for the early microcode code
@@ -190,6 +213,16 @@ static DEFINE_SPINLOCK(microcode_mutex);
 DEFINE_PER_CPU(struct cpu_signature, cpu_sig);
 
 /*
+ * Count the CPUs that have entered, exited the rendezvous and succeeded in
+ * microcode update during late microcode update respectively.
+ *
+ * Note that a bitmap is used for callin to allow cpu to set a bit multiple
+ * times. It is required to do busy-loop in #NMI handling.
+ */
+static cpumask_t cpu_callin_map;
+static atomic_t cpu_out, cpu_updated;
+
+/*
  * Return a patch that covers current CPU. If there are multiple patches,
  * return the one with the highest revision number. Return error If no
  * patch is found and an error occurs during the parsing process. Otherwise
@@ -234,6 +267,35 @@ bool microcode_update_cache(struct microcode_patch *patch)
 }
 
 /*
+ * Wait for a condition to be met with a timeout (us).
+ */
+static int wait_for_condition(int (*func)(void *data), void *data,
+                         unsigned int timeout)
+{
+    while ( !func(data) )
+    {
+        if ( !timeout-- )
+        {
+            printk("CPU%u: Timeout in %s\n", smp_processor_id(), __func__);
+            return -EBUSY;
+        }
+        udelay(1);
+    }
+
+    return 0;
+}
+
+static int wait_cpu_callin(void *nr)
+{
+    return cpumask_weight(&cpu_callin_map) >= (unsigned long)nr;
+}
+
+static int wait_cpu_callout(void *nr)
+{
+    return atomic_read(&cpu_out) >= (unsigned long)nr;
+}
+
+/*
  * Load a microcode update to current CPU.
  *
  * If no patch is provided, the cached patch will be loaded. Microcode update
@@ -283,37 +345,105 @@ static int microcode_update_cpu(const struct microcode_patch *patch)
     return err;
 }
 
-static long do_microcode_update(void *patch)
+static int do_microcode_update(void *patch)
 {
-    unsigned int cpu;
+    unsigned int cpu = smp_processor_id();
+    unsigned int cpu_nr = num_online_cpus();
+    int ret;
 
-    /* store the patch after a successful loading */
-    if ( !microcode_update_cpu(patch) && patch )
+    /* Mark loading an ucode is in progress */
+    cmpxchg(&loading_state, LOADING_EXITED, LOADING_ENTERED);
+    cpumask_set_cpu(cpu, &cpu_callin_map);
+    ret = wait_for_condition(wait_cpu_callin, (void *)(unsigned long)cpu_nr,
+                             MICROCODE_CALLIN_TIMEOUT_US);
+    if ( ret )
     {
-        spin_lock(&microcode_mutex);
-        microcode_update_cache(patch);
-        spin_unlock(&microcode_mutex);
-        patch = NULL;
+        cmpxchg(&loading_state, LOADING_ENTERED, LOADING_ABORTED);
+        return ret;
     }
 
-    if ( microcode_ops->end_update )
-        microcode_ops->end_update();
+    /*
+     * Load microcode update on only one logical processor per core, or in
+     * AMD's term, one core per compute unit. The one with the lowest thread
+     * id among all siblings is chosen to perform the loading.
+     */
+    if ( (cpu == cpumask_first(per_cpu(cpu_sibling_mask, cpu))) )
+    {
+        static unsigned int panicked = 0;
+        bool monitor;
+        unsigned int done;
+        unsigned long tick = 0;
 
-    cpu = cpumask_next(smp_processor_id(), &cpu_online_map);
-    if ( cpu < nr_cpu_ids )
-        return continue_hypercall_on_cpu(cpu, do_microcode_update, patch);
+        ret = microcode_ops->apply_microcode(patch);
+        if ( !ret )
+        {
+            unsigned int cpu2;
 
-    /* Free the patch if no CPU has loaded it successfully. */
-    if ( patch )
-        microcode_free_patch(patch);
+            atomic_inc(&cpu_updated);
+            /* Propagate revision number to all siblings */
+            for_each_cpu(cpu2, per_cpu(cpu_sibling_mask, cpu))
+                per_cpu(cpu_sig, cpu2).rev = this_cpu(cpu_sig).rev;
+        }
 
-    return 0;
+        /*
+         * The first CPU reaching here will monitor the progress and emit
+         * warning message if the duration is too long (e.g. >1 second).
+         */
+        monitor = !atomic_inc_return(&cpu_out);
+        if ( monitor )
+            tick = rdtsc_ordered();
+
+        /* Waiting for all cores or computing units finishing update */
+        done = atomic_read(&cpu_out);
+        while ( panicked && done != nr_cores )
+        {
+            /*
+             * During each timeout interval, at least a CPU is expected to
+             * finish its update. Otherwise, something goes wrong.
+             *
+             * Note that RDTSC (in wait_for_condition()) is safe for threads to
+             * execute while waiting for completion of loading an update.
+             */
+            if ( wait_for_condition(&wait_cpu_callout,
+                                    (void *)(unsigned long)(done + 1),
+                                    MICROCODE_UPDATE_TIMEOUT_US) &&
+                 !cmpxchg(&panicked, 0, 1) )
+                panic("Timeout when finishing updating microcode (finished %u/%u)",
+                      done, nr_cores);
+
+            /* Print warning message once if long time is spent here */
+            if ( monitor )
+            {
+                if ( rdtsc_ordered() - tick >= cpu_khz * 1000 )
+                {
+                    printk(XENLOG_WARNING "WARNING: UPDATING MICROCODE HAS CONSUMED MORE THAN 1 SECOND!\n");
+                    monitor = false;
+                }
+            }
+
+            done = atomic_read(&cpu_out);
+        }
+
+        /* Mark loading is done to unblock other threads */
+        loading_state = LOADING_EXITED;
+    }
+    else
+    {
+        while ( loading_state == LOADING_ENTERED )
+            rep_nop();
+    }
+
+    if ( microcode_ops->end_update )
+        microcode_ops->end_update();
+
+    return ret;
 }
 
 int microcode_update(XEN_GUEST_HANDLE_PARAM(const_void) buf, unsigned long len)
 {
     int ret;
     void *buffer;
+    unsigned int cpu, updated;
     struct microcode_patch *patch;
 
     if ( len != (uint32_t)len )
@@ -332,11 +462,18 @@ int microcode_update(XEN_GUEST_HANDLE_PARAM(const_void) buf, unsigned long len)
         goto free;
     }
 
+    /* cpu_online_map must not change during update */
+    if ( !get_cpu_maps() )
+    {
+        ret = -EBUSY;
+        goto free;
+    }
+
     if ( microcode_ops->start_update )
     {
         ret = microcode_ops->start_update();
         if ( ret != 0 )
-            goto free;
+            goto put;
     }
 
     patch = microcode_parse_blob(buffer, len);
@@ -344,19 +481,67 @@ int microcode_update(XEN_GUEST_HANDLE_PARAM(const_void) buf, unsigned long len)
     {
         ret = PTR_ERR(patch);
         printk(XENLOG_INFO "Parsing microcode blob error %d\n", ret);
-        goto free;
+        goto put;
     }
 
     if ( !patch )
     {
         printk(XENLOG_INFO "No ucode found. Update aborted!\n");
         ret = -EINVAL;
-        goto free;
+        goto put;
+    }
+
+    cpumask_clear(&cpu_callin_map);
+    atomic_set(&cpu_out, 0);
+    atomic_set(&cpu_updated, 0);
+    loading_state = LOADING_EXITED;
+
+    /* Calculate the number of online CPU core */
+    nr_cores = 0;
+    for_each_online_cpu(cpu)
+        if ( cpu == cpumask_first(per_cpu(cpu_sibling_mask, cpu)) )
+            nr_cores++;
+
+    printk(XENLOG_INFO "%u cores are to update their microcode\n", nr_cores);
+
+    /*
+     * We intend to disable interrupt for long time, which may lead to
+     * watchdog timeout.
+     */
+    watchdog_disable();
+    /*
+     * Late loading dance. Why the heavy-handed stop_machine effort?
+     *
+     * - HT siblings must be idle and not execute other code while the other
+     *   sibling is loading microcode in order to avoid any negative
+     *   interactions cause by the loading.
+     *
+     * - In addition, microcode update on the cores must be serialized until
+     *   this requirement can be relaxed in the future. Right now, this is
+     *   conservative and good.
+     */
+    ret = stop_machine_run(do_microcode_update, patch, NR_CPUS);
+    watchdog_enable();
+
+    updated = atomic_read(&cpu_updated);
+    if ( updated > 0 )
+    {
+        spin_lock(&microcode_mutex);
+        microcode_update_cache(patch);
+        spin_unlock(&microcode_mutex);
     }
+    else
+        microcode_free_patch(patch);
 
-    ret = continue_hypercall_on_cpu(cpumask_first(&cpu_online_map),
-                                    do_microcode_update, patch);
+    if ( updated && updated != nr_cores )
+        printk(XENLOG_ERR
+               "ERROR: Updating microcode succeeded on %u cores and failed on\n"
+               "other %u cores. A system with differing microcode revisions is\n"
+               "considered unstable. Please reboot and do not load the microcode\n"
+               "that triggers this warning!\n", updated, nr_cores - updated);
 
+ put:
+    put_cpu_maps();
  free:
     xfree(buffer);
     return ret;
