@@ -108,27 +108,36 @@ static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr)
unsigned long bfn = XEN_PFN_DOWN(dma_to_phys(dev, dma_addr));
unsigned long xen_pfn = bfn_to_local_pfn(bfn);
phys_addr_t paddr = (phys_addr_t)xen_pfn << XEN_PAGE_SHIFT;
+ int i;
/* If the address is outside our domain, it CAN
* have the same virtual address as another address
* in our domain. Therefore _only_ check address within our domain.
*/
- if (pfn_valid(PFN_DOWN(paddr))) {
- return paddr >= virt_to_phys(xen_io_tlb_start[SWIOTLB_LO]) &&
- paddr < virt_to_phys(xen_io_tlb_end[SWIOTLB_LO]);
- }
+ if (!pfn_valid(PFN_DOWN(paddr)))
+ return 0;
+
+ for (i = 0; i < swiotlb_nr; i++)
+ if (paddr >= virt_to_phys(xen_io_tlb_start[i]) &&
+ paddr < virt_to_phys(xen_io_tlb_end[i]))
+ return 1;
+
return 0;
}
static int
-xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs)
+xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs,
+ enum swiotlb_t type)
{
int i, rc;
int dma_bits;
dma_addr_t dma_handle;
phys_addr_t p = virt_to_phys(buf);
- dma_bits = get_order(IO_TLB_SEGSIZE << IO_TLB_SHIFT) + PAGE_SHIFT;
+ if (type == SWIOTLB_HI)
+ dma_bits = max_dma_bits[SWIOTLB_HI];
+ else
+ dma_bits = get_order(IO_TLB_SEGSIZE << IO_TLB_SHIFT) + PAGE_SHIFT;
i = 0;
do {
@@ -139,7 +148,7 @@ xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs)
p + (i << IO_TLB_SHIFT),
get_order(slabs << IO_TLB_SHIFT),
dma_bits, &dma_handle);
- } while (rc && dma_bits++ < max_dma_bits[SWIOTLB_LO]);
+ } while (rc && dma_bits++ < max_dma_bits[type]);
if (rc)
return rc;
@@ -147,16 +156,17 @@ xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs)
} while (i < nslabs);
return 0;
}
-static unsigned long xen_set_nslabs(unsigned long nr_tbl)
+
+static unsigned long xen_set_nslabs(unsigned long nr_tbl, enum swiotlb_t type)
{
if (!nr_tbl) {
- xen_io_tlb_nslabs[SWIOTLB_LO] = (64 * 1024 * 1024 >> IO_TLB_SHIFT);
- xen_io_tlb_nslabs[SWIOTLB_LO] = ALIGN(xen_io_tlb_nslabs[SWIOTLB_LO],
+ xen_io_tlb_nslabs[type] = (64 * 1024 * 1024 >> IO_TLB_SHIFT);
+ xen_io_tlb_nslabs[type] = ALIGN(xen_io_tlb_nslabs[type],
IO_TLB_SEGSIZE);
} else
- xen_io_tlb_nslabs[SWIOTLB_LO] = nr_tbl;
+ xen_io_tlb_nslabs[type] = nr_tbl;
- return xen_io_tlb_nslabs[SWIOTLB_LO] << IO_TLB_SHIFT;
+ return xen_io_tlb_nslabs[type] << IO_TLB_SHIFT;
}
enum xen_swiotlb_err {
@@ -180,23 +190,24 @@ static const char *xen_swiotlb_error(enum xen_swiotlb_err err)
}
return "";
}
-int __ref xen_swiotlb_init(int verbose, bool early)
+
+static int xen_swiotlb_init_type(int verbose, bool early, enum swiotlb_t type)
{
unsigned long bytes, order;
int rc = -ENOMEM;
enum xen_swiotlb_err m_ret = XEN_SWIOTLB_UNKNOWN;
unsigned int repeat = 3;
- xen_io_tlb_nslabs[SWIOTLB_LO] = swiotlb_nr_tbl(SWIOTLB_LO);
+ xen_io_tlb_nslabs[type] = swiotlb_nr_tbl(type);
retry:
- bytes = xen_set_nslabs(xen_io_tlb_nslabs[SWIOTLB_LO]);
- order = get_order(xen_io_tlb_nslabs[SWIOTLB_LO] << IO_TLB_SHIFT);
+ bytes = xen_set_nslabs(xen_io_tlb_nslabs[type], type);
+ order = get_order(xen_io_tlb_nslabs[type] << IO_TLB_SHIFT);
/*
* IO TLB memory already allocated. Just use it.
*/
- if (io_tlb_start[SWIOTLB_LO] != 0) {
- xen_io_tlb_start[SWIOTLB_LO] = phys_to_virt(io_tlb_start[SWIOTLB_LO]);
+ if (io_tlb_start[type] != 0) {
+ xen_io_tlb_start[type] = phys_to_virt(io_tlb_start[type]);
goto end;
}
@@ -204,81 +215,95 @@ int __ref xen_swiotlb_init(int verbose, bool early)
* Get IO TLB memory from any location.
*/
if (early) {
- xen_io_tlb_start[SWIOTLB_LO] = memblock_alloc(PAGE_ALIGN(bytes),
+ xen_io_tlb_start[type] = memblock_alloc(PAGE_ALIGN(bytes),
PAGE_SIZE);
- if (!xen_io_tlb_start[SWIOTLB_LO])
+ if (!xen_io_tlb_start[type])
panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
__func__, PAGE_ALIGN(bytes), PAGE_SIZE);
} else {
#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
- xen_io_tlb_start[SWIOTLB_LO] = (void *)xen_get_swiotlb_free_pages(order);
- if (xen_io_tlb_start[SWIOTLB_LO])
+ xen_io_tlb_start[type] = (void *)xen_get_swiotlb_free_pages(order);
+ if (xen_io_tlb_start[type])
break;
order--;
}
if (order != get_order(bytes)) {
pr_warn("Warning: only able to allocate %ld MB for software IO TLB\n",
(PAGE_SIZE << order) >> 20);
- xen_io_tlb_nslabs[SWIOTLB_LO] = SLABS_PER_PAGE << order;
- bytes = xen_io_tlb_nslabs[SWIOTLB_LO] << IO_TLB_SHIFT;
+ xen_io_tlb_nslabs[type] = SLABS_PER_PAGE << order;
+ bytes = xen_io_tlb_nslabs[type] << IO_TLB_SHIFT;
}
}
- if (!xen_io_tlb_start[SWIOTLB_LO]) {
+ if (!xen_io_tlb_start[type]) {
m_ret = XEN_SWIOTLB_ENOMEM;
goto error;
}
/*
* And replace that memory with pages under 4GB.
*/
- rc = xen_swiotlb_fixup(xen_io_tlb_start[SWIOTLB_LO],
+ rc = xen_swiotlb_fixup(xen_io_tlb_start[type],
bytes,
- xen_io_tlb_nslabs[SWIOTLB_LO]);
+ xen_io_tlb_nslabs[type],
+ type);
if (rc) {
if (early)
- memblock_free(__pa(xen_io_tlb_start[SWIOTLB_LO]),
+ memblock_free(__pa(xen_io_tlb_start[type]),
PAGE_ALIGN(bytes));
else {
- free_pages((unsigned long)xen_io_tlb_start[SWIOTLB_LO], order);
- xen_io_tlb_start[SWIOTLB_LO] = NULL;
+ free_pages((unsigned long)xen_io_tlb_start[type], order);
+ xen_io_tlb_start[type] = NULL;
}
m_ret = XEN_SWIOTLB_EFIXUP;
goto error;
}
if (early) {
- if (swiotlb_init_with_tbl(xen_io_tlb_start[SWIOTLB_LO],
- xen_io_tlb_nslabs[SWIOTLB_LO],
- SWIOTLB_LO, verbose))
+ if (swiotlb_init_with_tbl(xen_io_tlb_start[type],
+ xen_io_tlb_nslabs[type],
+ type, verbose))
panic("Cannot allocate SWIOTLB buffer");
rc = 0;
} else
- rc = swiotlb_late_init_with_tbl(xen_io_tlb_start[SWIOTLB_LO],
- xen_io_tlb_nslabs[SWIOTLB_LO],
- SWIOTLB_LO);
+ rc = swiotlb_late_init_with_tbl(xen_io_tlb_start[type],
+ xen_io_tlb_nslabs[type],
+ type);
end:
- xen_io_tlb_end[SWIOTLB_LO] = xen_io_tlb_start[SWIOTLB_LO] + bytes;
+ xen_io_tlb_end[type] = xen_io_tlb_start[type] + bytes;
if (!rc)
- swiotlb_set_max_segment(PAGE_SIZE, SWIOTLB_LO);
+ swiotlb_set_max_segment(PAGE_SIZE, type);
return rc;
error:
if (repeat--) {
- xen_io_tlb_nslabs[SWIOTLB_LO] = max(1024UL, /* Min is 2MB */
- (xen_io_tlb_nslabs[SWIOTLB_LO] >> 1));
+ xen_io_tlb_nslabs[type] = max(1024UL, /* Min is 2MB */
+ (xen_io_tlb_nslabs[type] >> 1));
pr_info("Lowering to %luMB\n",
- (xen_io_tlb_nslabs[SWIOTLB_LO] << IO_TLB_SHIFT) >> 20);
+ (xen_io_tlb_nslabs[type] << IO_TLB_SHIFT) >> 20);
goto retry;
}
pr_err("%s (rc:%d)\n", xen_swiotlb_error(m_ret), rc);
if (early)
panic("%s (rc:%d)", xen_swiotlb_error(m_ret), rc);
else
- free_pages((unsigned long)xen_io_tlb_start[SWIOTLB_LO], order);
+ free_pages((unsigned long)xen_io_tlb_start[type], order);
return rc;
}
+int __ref xen_swiotlb_init(int verbose, bool early)
+{
+ int i, rc;
+
+ for (i = 0; i < swiotlb_nr; i++) {
+ rc = xen_swiotlb_init_type(verbose, early, i);
+ if (rc)
+ return rc;
+ }
+
+ return 0;
+}
+
static void *
xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
@@ -566,7 +591,13 @@ xen_swiotlb_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
static int
xen_swiotlb_dma_supported(struct device *hwdev, u64 mask)
{
- return xen_virt_to_bus(hwdev, xen_io_tlb_end[SWIOTLB_LO] - 1) <= mask;
+ int i;
+
+ for (i = 0; i < swiotlb_nr; i++)
+ if (xen_virt_to_bus(hwdev, xen_io_tlb_end[i] - 1) <= mask)
+ return true;
+
+ return false;
}
const struct dma_map_ops xen_swiotlb_dma_ops = {
This patch is to enable the 64-bit xen-swiotlb buffer. For Xen PVM DMA address, the 64-bit device will be able to allocate from 64-bit swiotlb buffer. Cc: Joe Jin <joe.jin@oracle.com> Signed-off-by: Dongli Zhang <dongli.zhang@oracle.com> --- drivers/xen/swiotlb-xen.c | 117 ++++++++++++++++++++++++-------------- 1 file changed, 74 insertions(+), 43 deletions(-)