@@ -57,9 +57,6 @@
DEFINE_WD_CLASS(reservation_ww_class);
EXPORT_SYMBOL(reservation_ww_class);
-/* Mask for the lower fence pointer bits */
-#define DMA_RESV_LIST_MASK 0x3
-
struct dma_resv_list {
struct rcu_head rcu;
u32 num_fences, max_fences;
@@ -71,13 +68,14 @@ static void dma_resv_list_entry(struct dma_resv_list *list, unsigned int index,
struct dma_resv *resv, struct dma_fence **fence,
enum dma_resv_usage *usage)
{
- long tmp;
- tmp = (long)rcu_dereference_check(list->table[index],
- resv ? dma_resv_held(resv) : true);
- *fence = (struct dma_fence *)(tmp & ~DMA_RESV_LIST_MASK);
+ *fence = (struct dma_fence *)rcu_dereference_check(list->table[index],
+ resv ?
+ dma_resv_held(resv) :
+ true);
+
if (usage)
- *usage = tmp & DMA_RESV_LIST_MASK;
+ *usage = (*fence)->usage;
}
/* Set the fence and usage flags at the specific index in the list. */
@@ -86,9 +84,8 @@ static void dma_resv_list_set(struct dma_resv_list *list,
struct dma_fence *fence,
enum dma_resv_usage usage)
{
- long tmp = ((long)fence) | usage;
-
- RCU_INIT_POINTER(list->table[index], (struct dma_fence *)tmp);
+ fence->usage = usage;
+ RCU_INIT_POINTER(list->table[index], fence);
}
/*
@@ -527,7 +524,7 @@ int dma_resv_copy_fences(struct dma_resv *dst, struct dma_resv *src)
list = NULL;
- dma_resv_iter_begin(&cursor, src, DMA_RESV_USAGE_BOOKKEEP);
+ dma_resv_iter_begin(&cursor, src, DMA_RESV_USAGE_PREEMPT);
dma_resv_for_each_fence_unlocked(&cursor, f) {
if (dma_resv_iter_is_restarted(&cursor)) {
@@ -680,8 +677,11 @@ long dma_resv_wait_timeout(struct dma_resv *obj, enum dma_resv_usage usage,
long ret = timeout ? timeout : 1;
struct dma_resv_iter cursor;
struct dma_fence *fence;
+ enum dma_resv_usage walk_usage = (usage == DMA_RESV_USAGE_PREEMPT) ?
+ DMA_RESV_USAGE_BOOKKEEP : usage;
- dma_resv_iter_begin(&cursor, obj, usage);
+wait_again:
+ dma_resv_iter_begin(&cursor, obj, walk_usage);
dma_resv_for_each_fence_unlocked(&cursor, fence) {
ret = dma_fence_wait_timeout(fence, intr, ret);
@@ -692,6 +692,20 @@ long dma_resv_wait_timeout(struct dma_resv *obj, enum dma_resv_usage usage,
}
dma_resv_iter_end(&cursor);
+ /*
+ * Now wait on preempt fences, pipeline preemption by enabling signaling
+ * before waiting.
+ */
+ if (walk_usage != usage) {
+ dma_resv_iter_begin(&cursor, obj, usage);
+ dma_resv_for_each_fence_unlocked(&cursor, fence)
+ dma_fence_enable_sw_signaling(fence);
+ dma_resv_iter_end(&cursor);
+
+ walk_usage = usage;
+ goto wait_again;
+ }
+
return ret;
}
EXPORT_SYMBOL_GPL(dma_resv_wait_timeout);
@@ -757,7 +771,8 @@ EXPORT_SYMBOL_GPL(dma_resv_test_signaled);
*/
void dma_resv_describe(struct dma_resv *obj, struct seq_file *seq)
{
- static const char *usage[] = { "kernel", "write", "read", "bookkeep" };
+ static const char *usage[] = { "kernel", "write", "read", "bookkeep",
+ "preempt" };
struct dma_resv_iter cursor;
struct dma_fence *fence;
@@ -306,7 +306,7 @@ int dma_resv(void)
int r;
spin_lock_init(&fence_lock);
- for (usage = DMA_RESV_USAGE_KERNEL; usage <= DMA_RESV_USAGE_BOOKKEEP;
+ for (usage = DMA_RESV_USAGE_KERNEL; usage <= DMA_RESV_USAGE_PREEMPT;
++usage) {
r = subtests(tests, (void *)(unsigned long)usage);
if (r)
@@ -94,6 +94,7 @@ struct dma_fence {
unsigned long flags;
struct kref refcount;
int error;
+ u8 usage;
};
enum dma_fence_flag_bits {
@@ -55,9 +55,10 @@ struct dma_resv_list;
* This enum describes the different use cases for a dma_resv object and
* controls which fences are returned when queried.
*
- * An important fact is that there is the order KERNEL<WRITE<READ<BOOKKEEP and
- * when the dma_resv object is asked for fences for one use case the fences
- * for the lower use case are returned as well.
+ * An important fact is that there is the order
+ * KERNEL<WRITE<READ<BOOKKEEP<PREEMPT and when the dma_resv object is asked for
+ * fences for one use case the fences for the lower use case are returned as
+ * well.
*
* For example when asking for WRITE fences then the KERNEL fences are returned
* as well. Similar when asked for READ fences then both WRITE and KERNEL
@@ -105,15 +106,26 @@ enum dma_resv_usage {
* This should be used by submissions which don't want to participate in
* any implicit synchronization.
*
- * The most common cases are preemption fences, page table updates, TLB
- * flushes as well as explicitly synced user submissions.
+ * The most common cases are page table updates, TLB flushes as well as
+ * explicitly synced user submissions.
*
* Explicitly synced user submissions can be promoted to
* DMA_RESV_USAGE_READ or DMA_RESV_USAGE_WRITE as needed using
* dma_buf_import_sync_file() when implicit synchronization should
* become necessary after initial adding of the fence.
*/
- DMA_RESV_USAGE_BOOKKEEP
+ DMA_RESV_USAGE_BOOKKEEP,
+
+ /**
+ * @DMA_RESV_USAGE_PREEMPT: Preempt.
+ *
+ * This kernel-owned slot is to install preempt fences. The semantics
+ * require that enabling signaling on a preemption fence (and thus
+ * preempting device execution) only occurs once all other fences in the
+ * reservation are signaled. These rules are enforced by dma-resv to
+ * ensure correct usage.
+ */
+ DMA_RESV_USAGE_PREEMPT,
};
/**
Add a new DMA reservation slot, which is the highest priority and used exclusively for preemptive fences. The semantics of this slot require that all other fences in other slots must be signaled before any fences in the preemptive slot enable signaling. These semantics exist to avoid deadlocks during user submissions when using DMA fences. These rules are baked into dma-resv. Cc: Dave Airlie <airlied@redhat.com> Cc: Simona Vetter <simona.vetter@ffwll.ch> Cc: Christian Koenig <christian.koenig@amd.com> Signed-off-by: Matthew Brost <matthew.brost@intel.com> --- drivers/dma-buf/dma-resv.c | 43 +++++++++++++++++++++++------------ drivers/dma-buf/st-dma-resv.c | 2 +- include/linux/dma-fence.h | 1 + include/linux/dma-resv.h | 24 ++++++++++++++----- 4 files changed, 49 insertions(+), 21 deletions(-)