@@ -69,6 +69,7 @@ typedef struct QemuDsaBatchTask {
QemuDsaTaskType task_type;
QemuDsaTaskStatus status;
int batch_size;
+ bool *results;
QSIMPLEQ_ENTRY(QemuDsaBatchTask) entry;
} QemuDsaBatchTask;
@@ -33,9 +33,20 @@
#define DSA_WQ_PORTAL_SIZE 4096
#define DSA_WQ_DEPTH 128
#define MAX_DSA_DEVICES 16
+#define DSA_COMPLETION_THREAD "qemu_dsa_completion"
+
+typedef struct {
+ bool stopping;
+ bool running;
+ QemuThread thread;
+ int thread_id;
+ QemuSemaphore sem_init_done;
+ QemuDsaDeviceGroup *group;
+} QemuDsaCompletionThread;
uint32_t max_retry_count;
static QemuDsaDeviceGroup dsa_group;
+static QemuDsaCompletionThread completion_thread;
/**
@@ -387,6 +398,263 @@ submit_batch_wi_async(QemuDsaBatchTask *batch_task)
return dsa_task_enqueue(device_group, batch_task);
}
+/**
+ * @brief Poll for the DSA work item completion.
+ *
+ * @param completion A pointer to the DSA work item completion record.
+ * @param opcode The DSA opcode.
+ *
+ * @return Zero if successful, non-zero otherwise.
+ */
+static int
+poll_completion(struct dsa_completion_record *completion,
+ enum dsa_opcode opcode)
+{
+ uint8_t status;
+ uint64_t retry = 0;
+
+ while (true) {
+ /* The DSA operation completes successfully or fails. */
+ status = completion->status;
+ if (status == DSA_COMP_SUCCESS ||
+ status == DSA_COMP_PAGE_FAULT_NOBOF ||
+ status == DSA_COMP_BATCH_PAGE_FAULT ||
+ status == DSA_COMP_BATCH_FAIL) {
+ break;
+ } else if (status != DSA_COMP_NONE) {
+ error_report("DSA opcode %d failed with status = %d.",
+ opcode, status);
+ return 1;
+ }
+ retry++;
+ if (retry > max_retry_count) {
+ error_report("DSA wait for completion retry %lu times.", retry);
+ return 1;
+ }
+ _mm_pause();
+ }
+
+ return 0;
+}
+
+/**
+ * @brief Complete a single DSA task in the batch task.
+ *
+ * @param task A pointer to the batch task structure.
+ *
+ * @return Zero if successful, otherwise non-zero.
+ */
+static int
+poll_task_completion(QemuDsaBatchTask *task)
+{
+ assert(task->task_type == QEMU_DSA_TASK);
+
+ struct dsa_completion_record *completion = &task->completions[0];
+ uint8_t status;
+ int ret;
+
+ ret = poll_completion(completion, task->descriptors[0].opcode);
+ if (ret != 0) {
+ goto exit;
+ }
+
+ status = completion->status;
+ if (status == DSA_COMP_SUCCESS) {
+ task->results[0] = (completion->result == 0);
+ goto exit;
+ }
+
+ assert(status == DSA_COMP_PAGE_FAULT_NOBOF);
+
+exit:
+ return ret;
+}
+
+/**
+ * @brief Poll a batch task status until it completes. If DSA task doesn't
+ * complete properly, use CPU to complete the task.
+ *
+ * @param batch_task A pointer to the DSA batch task.
+ *
+ * @return Zero if successful, otherwise non-zero.
+ */
+static int
+poll_batch_task_completion(QemuDsaBatchTask *batch_task)
+{
+ struct dsa_completion_record *batch_completion =
+ &batch_task->batch_completion;
+ struct dsa_completion_record *completion;
+ uint8_t batch_status;
+ uint8_t status;
+ bool *results = batch_task->results;
+ uint32_t count = batch_task->batch_descriptor.desc_count;
+ int ret;
+
+ ret = poll_completion(batch_completion,
+ batch_task->batch_descriptor.opcode);
+ if (ret != 0) {
+ goto exit;
+ }
+
+ batch_status = batch_completion->status;
+
+ if (batch_status == DSA_COMP_SUCCESS) {
+ if (batch_completion->bytes_completed == count) {
+ /*
+ * Let's skip checking for each descriptors' completion status
+ * if the batch descriptor says all succedded.
+ */
+ for (int i = 0; i < count; i++) {
+ assert(batch_task->completions[i].status == DSA_COMP_SUCCESS);
+ results[i] = (batch_task->completions[i].result == 0);
+ }
+ goto exit;
+ }
+ } else {
+ assert(batch_status == DSA_COMP_BATCH_FAIL ||
+ batch_status == DSA_COMP_BATCH_PAGE_FAULT);
+ }
+
+ for (int i = 0; i < count; i++) {
+
+ completion = &batch_task->completions[i];
+ status = completion->status;
+
+ if (status == DSA_COMP_SUCCESS) {
+ results[i] = (completion->result == 0);
+ continue;
+ }
+
+ if (status != DSA_COMP_PAGE_FAULT_NOBOF) {
+ error_report("Unexpected DSA completion status = %u.", status);
+ ret = 1;
+ goto exit;
+ }
+ }
+
+exit:
+ return ret;
+}
+
+/**
+ * @brief Handles an asynchronous DSA batch task completion.
+ *
+ * @param task A pointer to the batch buffer zero task structure.
+ */
+static void
+dsa_batch_task_complete(QemuDsaBatchTask *batch_task)
+{
+ batch_task->status = QEMU_DSA_TASK_COMPLETION;
+ batch_task->completion_callback(batch_task);
+}
+
+/**
+ * @brief The function entry point called by a dedicated DSA
+ * work item completion thread.
+ *
+ * @param opaque A pointer to the thread context.
+ *
+ * @return void* Not used.
+ */
+static void *
+dsa_completion_loop(void *opaque)
+{
+ QemuDsaCompletionThread *thread_context =
+ (QemuDsaCompletionThread *)opaque;
+ QemuDsaBatchTask *batch_task;
+ QemuDsaDeviceGroup *group = thread_context->group;
+ int ret;
+
+ rcu_register_thread();
+
+ thread_context->thread_id = qemu_get_thread_id();
+ qemu_sem_post(&thread_context->sem_init_done);
+
+ while (thread_context->running) {
+ batch_task = dsa_task_dequeue(group);
+ assert(batch_task != NULL || !group->running);
+ if (!group->running) {
+ assert(!thread_context->running);
+ break;
+ }
+ if (batch_task->task_type == QEMU_DSA_TASK) {
+ ret = poll_task_completion(batch_task);
+ } else {
+ assert(batch_task->task_type == QEMU_DSA_BATCH_TASK);
+ ret = poll_batch_task_completion(batch_task);
+ }
+
+ if (ret != 0) {
+ goto exit;
+ }
+
+ dsa_batch_task_complete(batch_task);
+ }
+
+exit:
+ if (ret != 0) {
+ error_report("DSA completion thread exited due to internal error.");
+ }
+ rcu_unregister_thread();
+ return NULL;
+}
+
+/**
+ * @brief Initializes a DSA completion thread.
+ *
+ * @param completion_thread A pointer to the completion thread context.
+ * @param group A pointer to the DSA device group.
+ */
+static void
+dsa_completion_thread_init(
+ QemuDsaCompletionThread *completion_thread,
+ QemuDsaDeviceGroup *group)
+{
+ completion_thread->stopping = false;
+ completion_thread->running = true;
+ completion_thread->thread_id = -1;
+ qemu_sem_init(&completion_thread->sem_init_done, 0);
+ completion_thread->group = group;
+
+ qemu_thread_create(&completion_thread->thread,
+ DSA_COMPLETION_THREAD,
+ dsa_completion_loop,
+ completion_thread,
+ QEMU_THREAD_JOINABLE);
+
+ /* Wait for initialization to complete */
+ qemu_sem_wait(&completion_thread->sem_init_done);
+}
+
+/**
+ * @brief Stops the completion thread (and implicitly, the device group).
+ *
+ * @param opaque A pointer to the completion thread.
+ */
+static void dsa_completion_thread_stop(void *opaque)
+{
+ QemuDsaCompletionThread *thread_context =
+ (QemuDsaCompletionThread *)opaque;
+
+ QemuDsaDeviceGroup *group = thread_context->group;
+
+ qemu_mutex_lock(&group->task_queue_lock);
+
+ thread_context->stopping = true;
+ thread_context->running = false;
+
+ /* Prevent the compiler from setting group->running first. */
+ barrier();
+ dsa_device_group_stop(group);
+
+ qemu_cond_signal(&group->task_queue_cond);
+ qemu_mutex_unlock(&group->task_queue_lock);
+
+ qemu_thread_join(&thread_context->thread);
+
+ qemu_sem_destroy(&thread_context->sem_init_done);
+}
+
/**
* @brief Check if DSA is running.
*
@@ -394,7 +662,7 @@ submit_batch_wi_async(QemuDsaBatchTask *batch_task)
*/
bool qemu_dsa_is_running(void)
{
- return false;
+ return completion_thread.running;
}
static void
@@ -435,6 +703,7 @@ void qemu_dsa_start(void)
return;
}
dsa_device_group_start(&dsa_group);
+ dsa_completion_thread_init(&completion_thread, &dsa_group);
}
/**
@@ -449,6 +718,7 @@ void qemu_dsa_stop(void)
return;
}
+ dsa_completion_thread_stop(&completion_thread);
dsa_empty_task_queue(group);
}