| Message ID | 20240515071057.33990-22-clement.mathieu--drif@eviden.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | ATS support for VT-d | expand |
>-----Original Message----- >From: CLEMENT MATHIEU--DRIF <clement.mathieu--drif@eviden.com> >Subject: [PATCH ats_vtd v2 21/25] atc: generic ATC that can be used by PCIe >devices that support SVM > >As the SVM-capable devices will need to cache translations, we provide >an first implementation. > >This cache uses a two-level design based on hash tables. >The first level is indexed by a PASID and the second by a virtual addresse. > >Signed-off-by: Clément Mathieu--Drif <clement.mathieu--drif@eviden.com> >--- > tests/unit/meson.build | 1 + > tests/unit/test-atc.c | 502 >+++++++++++++++++++++++++++++++++++++++++ > util/atc.c | 211 +++++++++++++++++ > util/atc.h | 117 ++++++++++ > util/meson.build | 1 + > 5 files changed, 832 insertions(+) > create mode 100644 tests/unit/test-atc.c > create mode 100644 util/atc.c > create mode 100644 util/atc.h Maybe the unit test can be split from functional change? > >diff --git a/tests/unit/meson.build b/tests/unit/meson.build >index 228a21d03c..5c9a6fe9f4 100644 >--- a/tests/unit/meson.build >+++ b/tests/unit/meson.build >@@ -52,6 +52,7 @@ tests = { > 'test-interval-tree': [], > 'test-xs-node': [qom], > 'test-virtio-dmabuf': [meson.project_source_root() / 'hw/display/virtio- >dmabuf.c'], >+ 'test-atc': [] > } > > if have_system or have_tools >diff --git a/tests/unit/test-atc.c b/tests/unit/test-atc.c >new file mode 100644 >index 0000000000..60fa60924a >--- /dev/null >+++ b/tests/unit/test-atc.c >@@ -0,0 +1,502 @@ >+/* >+ * This program is free software; you can redistribute it and/or modify >+ * it under the terms of the GNU General Public License as published by >+ * the Free Software Foundation; either version 2 of the License, or >+ * (at your option) any later version. >+ >+ * This program is distributed in the hope that it will be useful, >+ * but WITHOUT ANY WARRANTY; without even the implied warranty of >+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >+ * GNU General Public License for more details. >+ >+ * You should have received a copy of the GNU General Public License along >+ * with this program; if not, see <http://www.gnu.org/licenses/>. >+ */ >+ >+#include "util/atc.h" >+ >+static inline bool tlb_entry_equal(IOMMUTLBEntry *e1, IOMMUTLBEntry >*e2) >+{ >+ if (!e1 || !e2) { >+ return !e1 && !e2; >+ } >+ return e1->iova == e2->iova && >+ e1->addr_mask == e2->addr_mask && >+ e1->pasid == e2->pasid && >+ e1->perm == e2->perm && >+ e1->target_as == e2->target_as && >+ e1->translated_addr == e2->translated_addr; >+} >+ >+static void assert_lookup_equals(ATC *atc, IOMMUTLBEntry *target, >+ uint32_t pasid, hwaddr iova) >+{ >+ IOMMUTLBEntry *result; >+ result = atc_lookup(atc, pasid, iova); >+ g_assert(tlb_entry_equal(result, target)); >+} >+ >+static void check_creation(uint64_t page_size, uint8_t address_width, >+ uint8_t levels, uint8_t level_offset, >+ bool should_work) { >+ ATC *atc = atc_new(page_size, address_width); >+ if (atc) { >+ if (atc->levels != levels || atc->level_offset != level_offset) { >+ g_assert(false); /* ATC created but invalid configuration : fail */ >+ } >+ atc_destroy(atc); >+ g_assert(should_work); >+ } else { >+ g_assert(!should_work); >+ } >+} >+ >+static void test_creation_parameters(void) >+{ >+ check_creation(8, 39, 3, 9, false); >+ check_creation(4095, 39, 3, 9, false); >+ check_creation(4097, 39, 3, 9, false); >+ check_creation(8192, 48, 0, 0, false); >+ >+ check_creation(4096, 38, 0, 0, false); >+ check_creation(4096, 39, 3, 9, true); >+ check_creation(4096, 40, 0, 0, false); >+ check_creation(4096, 47, 0, 0, false); >+ check_creation(4096, 48, 4, 9, true); >+ check_creation(4096, 49, 0, 0, false); >+ check_creation(4096, 56, 0, 0, false); >+ check_creation(4096, 57, 5, 9, true); >+ check_creation(4096, 58, 0, 0, false); >+ >+ check_creation(16384, 35, 0, 0, false); >+ check_creation(16384, 36, 2, 11, true); >+ check_creation(16384, 37, 0, 0, false); >+ check_creation(16384, 46, 0, 0, false); >+ check_creation(16384, 47, 3, 11, true); >+ check_creation(16384, 48, 0, 0, false); >+ check_creation(16384, 57, 0, 0, false); >+ check_creation(16384, 58, 4, 11, true); >+ check_creation(16384, 59, 0, 0, false); >+} >+ >+static void test_single_entry(void) >+{ >+ IOMMUTLBEntry entry = { >+ .iova = 0x123456789000ULL, >+ .addr_mask = 0xfffULL, >+ .pasid = 5, >+ .perm = IOMMU_RW, >+ .translated_addr = 0xdeadbeefULL, >+ }; >+ >+ ATC *atc = atc_new(4096, 48); >+ g_assert(atc); >+ >+ assert_lookup_equals(atc, NULL, entry.pasid, >+ entry.iova + (entry.addr_mask / 2)); >+ >+ atc_create_address_space_cache(atc, entry.pasid); >+ g_assert(atc_update(atc, &entry) == 0); >+ >+ assert_lookup_equals(atc, NULL, entry.pasid + 1, >+ entry.iova + (entry.addr_mask / 2)); >+ assert_lookup_equals(atc, &entry, entry.pasid, >+ entry.iova + (entry.addr_mask / 2)); >+ >+ atc_destroy(atc); >+} >+ >+static void test_page_boundaries(void) >+{ >+ static const uint32_t pasid = 5; >+ static const hwaddr page_size = 4096; >+ >+ /* 2 consecutive entries */ >+ IOMMUTLBEntry e1 = { >+ .iova = 0x123456789000ULL, >+ .addr_mask = page_size - 1, >+ .pasid = pasid, >+ .perm = IOMMU_RW, >+ .translated_addr = 0xdeadbeefULL, >+ }; >+ IOMMUTLBEntry e2 = { >+ .iova = e1.iova + page_size, >+ .addr_mask = page_size - 1, >+ .pasid = pasid, >+ .perm = IOMMU_RW, >+ .translated_addr = 0x900df00dULL, >+ }; >+ >+ ATC *atc = atc_new(page_size, 48); >+ >+ atc_create_address_space_cache(atc, e1.pasid); >+ /* creating the address space twice should not be a problem */ >+ atc_create_address_space_cache(atc, e1.pasid); >+ >+ atc_update(atc, &e1); >+ atc_update(atc, &e2); >+ >+ assert_lookup_equals(atc, NULL, e1.pasid, e1.iova - 1); >+ assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >+ assert_lookup_equals(atc, &e1, e1.pasid, e1.iova + e1.addr_mask); >+ g_assert((e1.iova + e1.addr_mask + 1) == e2.iova); >+ assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >+ assert_lookup_equals(atc, &e2, e2.pasid, e2.iova + e2.addr_mask); >+ assert_lookup_equals(atc, NULL, e2.pasid, e2.iova + e2.addr_mask + 1); >+ >+ assert_lookup_equals(atc, NULL, e1.pasid + 10, e1.iova); >+ assert_lookup_equals(atc, NULL, e2.pasid + 10, e2.iova); >+ atc_destroy(atc); >+} >+ >+static void test_huge_page(void) >+{ >+ static const uint32_t pasid = 5; >+ static const hwaddr page_size = 4096; >+ IOMMUTLBEntry e1 = { >+ .iova = 0x123456600000ULL, >+ .addr_mask = 0x1fffffULL, >+ .pasid = pasid, >+ .perm = IOMMU_RW, >+ .translated_addr = 0xdeadbeefULL, >+ }; >+ hwaddr addr; >+ >+ ATC *atc = atc_new(page_size, 48); >+ >+ atc_create_address_space_cache(atc, e1.pasid); >+ atc_update(atc, &e1); >+ >+ for (addr = e1.iova; addr <= e1.iova + e1.addr_mask; addr += page_size) { >+ assert_lookup_equals(atc, &e1, e1.pasid, addr); >+ } >+ /* addr is now out of the huge page */ >+ assert_lookup_equals(atc, NULL, e1.pasid, addr); >+ atc_destroy(atc); >+} >+ >+static void test_pasid(void) >+{ >+ hwaddr addr = 0xaaaaaaaaa000ULL; >+ IOMMUTLBEntry e1 = { >+ .iova = addr, >+ .addr_mask = 0xfffULL, >+ .pasid = 8, >+ .perm = IOMMU_RW, >+ .translated_addr = 0xdeadbeefULL, >+ }; >+ IOMMUTLBEntry e2 = { >+ .iova = addr, >+ .addr_mask = 0xfffULL, >+ .pasid = 2, >+ .perm = IOMMU_RW, >+ .translated_addr = 0xb001ULL, >+ }; >+ uint16_t i; >+ >+ ATC *atc = atc_new(4096, 48); >+ >+ atc_create_address_space_cache(atc, e1.pasid); >+ atc_create_address_space_cache(atc, e2.pasid); >+ atc_update(atc, &e1); >+ atc_update(atc, &e2); >+ >+ for (i = 0; i <= MAX(e1.pasid, e2.pasid) + 1; ++i) { >+ if (i == e1.pasid || i == e2.pasid) { >+ continue; >+ } >+ assert_lookup_equals(atc, NULL, i, addr); >+ } >+ assert_lookup_equals(atc, &e1, e1.pasid, addr); >+ assert_lookup_equals(atc, &e1, e1.pasid, addr); >+ atc_destroy(atc); >+} >+ >+static void test_large_address(void) >+{ >+ IOMMUTLBEntry e1 = { >+ .iova = 0xaaaaaaaaa000ULL, >+ .addr_mask = 0xfffULL, >+ .pasid = 8, >+ .perm = IOMMU_RW, >+ .translated_addr = 0x5eeeeeedULL, >+ }; >+ IOMMUTLBEntry e2 = { >+ .iova = 0x1f00baaaaabf000ULL, >+ .addr_mask = 0xfffULL, >+ .pasid = e1.pasid, >+ .perm = IOMMU_RW, >+ .translated_addr = 0xdeadbeefULL, >+ }; >+ >+ ATC *atc = atc_new(4096, 57); >+ >+ atc_create_address_space_cache(atc, e1.pasid); >+ atc_update(atc, &e1); >+ atc_update(atc, &e2); >+ >+ assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >+ assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >+ atc_destroy(atc); >+} >+ >+static void test_bigger_page(void) >+{ >+ IOMMUTLBEntry e1 = { >+ .iova = 0xaabbccdde000ULL, >+ .addr_mask = 0x1fffULL, >+ .pasid = 1, >+ .perm = IOMMU_RW, >+ .translated_addr = 0x5eeeeeedULL, >+ }; >+ hwaddr i; >+ >+ ATC *atc = atc_new(8192, 43); >+ >+ atc_create_address_space_cache(atc, e1.pasid); >+ atc_update(atc, &e1); >+ >+ i = e1.iova & (~e1.addr_mask); >+ assert_lookup_equals(atc, NULL, e1.pasid, i - 1); >+ while (i <= e1.iova + e1.addr_mask) { >+ assert_lookup_equals(atc, &e1, e1.pasid, i); >+ ++i; >+ } >+ assert_lookup_equals(atc, NULL, e1.pasid, i); >+ atc_destroy(atc); >+} >+ >+static void test_unknown_pasid(void) >+{ >+ IOMMUTLBEntry e1 = { >+ .iova = 0xaabbccfff000ULL, >+ .addr_mask = 0xfffULL, >+ .pasid = 1, >+ .perm = IOMMU_RW, >+ .translated_addr = 0x5eeeeeedULL, >+ }; >+ >+ ATC *atc = atc_new(4096, 48); >+ g_assert(atc_update(atc, &e1) != 0); >+ assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); >+} >+ >+static void test_invalidation(void) >+{ >+ static uint64_t page_size = 4096; >+ IOMMUTLBEntry e1 = { >+ .iova = 0xaabbccddf000ULL, >+ .addr_mask = 0xfffULL, >+ .pasid = 1, >+ .perm = IOMMU_RW, >+ .translated_addr = 0x5eeeeeedULL, >+ }; >+ IOMMUTLBEntry e2 = { >+ .iova = 0xffe00000ULL, >+ .addr_mask = 0x1fffffULL, >+ .pasid = 1, >+ .perm = IOMMU_RW, >+ .translated_addr = 0xb000001ULL, >+ }; >+ IOMMUTLBEntry e3; >+ >+ ATC *atc = atc_new(page_size , 48); >+ atc_create_address_space_cache(atc, e1.pasid); >+ >+ atc_update(atc, &e1); >+ assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >+ atc_invalidate(atc, &e1); >+ assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); >+ >+ atc_update(atc, &e1); >+ atc_update(atc, &e2); >+ assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >+ assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >+ atc_invalidate(atc, &e2); >+ assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >+ assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); >+ >+ /* invalidate a huge page by invalidating a small region */ >+ for (hwaddr addr = e2.iova; addr <= (e2.iova + e2.addr_mask); >+ addr += page_size) { >+ atc_update(atc, &e2); >+ assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >+ e3 = (IOMMUTLBEntry){ >+ .iova = addr, >+ .addr_mask = page_size - 1, >+ .pasid = e2.pasid, >+ .perm = IOMMU_RW, >+ .translated_addr = 0, >+ }; >+ atc_invalidate(atc, &e3); >+ assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); >+ } >+} >+ >+static void test_delete_address_space_cache(void) >+{ >+ static uint64_t page_size = 4096; >+ IOMMUTLBEntry e1 = { >+ .iova = 0xaabbccddf000ULL, >+ .addr_mask = 0xfffULL, >+ .pasid = 1, >+ .perm = IOMMU_RW, >+ .translated_addr = 0x5eeeeeedULL, >+ }; >+ IOMMUTLBEntry e2 = { >+ .iova = e1.iova, >+ .addr_mask = 0xfffULL, >+ .pasid = 2, >+ .perm = IOMMU_RW, >+ .translated_addr = 0x5eeeeeedULL, >+ }; >+ >+ ATC *atc = atc_new(page_size , 48); >+ atc_create_address_space_cache(atc, e1.pasid); >+ >+ atc_update(atc, &e1); >+ assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >+ atc_invalidate(atc, &e2); /* unkown pasid : is a nop*/ >+ assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >+ >+ atc_create_address_space_cache(atc, e2.pasid); >+ atc_update(atc, &e2); >+ assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >+ assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >+ atc_invalidate(atc, &e1); >+ /* e1 has been removed but e2 is still there */ >+ assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); >+ assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >+ >+ atc_update(atc, &e1); >+ assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >+ assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >+ >+ atc_delete_address_space_cache(atc, e2.pasid); >+ assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >+ assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); >+} >+ >+static void test_invalidate_entire_address_space(void) >+{ >+ static uint64_t page_size = 4096; >+ IOMMUTLBEntry e1 = { >+ .iova = 0x1000ULL, >+ .addr_mask = 0xfffULL, >+ .pasid = 1, >+ .perm = IOMMU_RW, >+ .translated_addr = 0x5eedULL, >+ }; >+ IOMMUTLBEntry e2 = { >+ .iova = 0xfffffffff000ULL, >+ .addr_mask = 0xfffULL, >+ .pasid = 1, >+ .perm = IOMMU_RW, >+ .translated_addr = 0xbeefULL, >+ }; >+ IOMMUTLBEntry e3 = { >+ .iova = 0, >+ .addr_mask = 0xffffffffffffffffULL, >+ .pasid = 1, >+ .perm = IOMMU_RW, >+ .translated_addr = 0, >+ }; >+ >+ ATC *atc = atc_new(page_size , 48); >+ atc_create_address_space_cache(atc, e1.pasid); >+ >+ atc_update(atc, &e1); >+ atc_update(atc, &e2); >+ assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >+ assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >+ atc_invalidate(atc, &e3); >+ /* e1 has been removed but e2 is still there */ >+ assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); >+ assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); >+ >+ atc_destroy(atc); >+} >+ >+static void test_reset(void) >+{ >+ static uint64_t page_size = 4096; >+ IOMMUTLBEntry e1 = { >+ .iova = 0x1000ULL, >+ .addr_mask = 0xfffULL, >+ .pasid = 1, >+ .perm = IOMMU_RW, >+ .translated_addr = 0x5eedULL, >+ }; >+ IOMMUTLBEntry e2 = { >+ .iova = 0xfffffffff000ULL, >+ .addr_mask = 0xfffULL, >+ .pasid = 2, >+ .perm = IOMMU_RW, >+ .translated_addr = 0xbeefULL, >+ }; >+ >+ ATC *atc = atc_new(page_size , 48); >+ atc_create_address_space_cache(atc, e1.pasid); >+ atc_create_address_space_cache(atc, e2.pasid); >+ atc_update(atc, &e1); >+ atc_update(atc, &e2); >+ >+ assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >+ assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >+ >+ atc_reset(atc); >+ >+ assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); >+ assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); >+} >+ >+static void test_get_max_number_of_pages(void) >+{ >+ static uint64_t page_size = 4096; >+ hwaddr base = 0xc0fee000; /* aligned */ >+ ATC *atc = atc_new(page_size , 48); >+ g_assert(atc_get_max_number_of_pages(atc, base, page_size / 2) == 1); >+ g_assert(atc_get_max_number_of_pages(atc, base, page_size) == 1); >+ g_assert(atc_get_max_number_of_pages(atc, base, page_size + 1) == 2); >+ >+ g_assert(atc_get_max_number_of_pages(atc, base + 10, 1) == 1); >+ g_assert(atc_get_max_number_of_pages(atc, base + 10, page_size - 10) >== 1); >+ g_assert(atc_get_max_number_of_pages(atc, base + 10, >+ page_size - 10 + 1) == 2); >+ g_assert(atc_get_max_number_of_pages(atc, base + 10, >+ page_size - 10 + 2) == 2); >+ >+ g_assert(atc_get_max_number_of_pages(atc, base + page_size - 1, 1) == >1); >+ g_assert(atc_get_max_number_of_pages(atc, base + page_size - 1, 2) == >2); >+ g_assert(atc_get_max_number_of_pages(atc, base + page_size - 1, 3) == >2); >+ >+ g_assert(atc_get_max_number_of_pages(atc, base + 10, page_size * 20) >== 21); >+ g_assert(atc_get_max_number_of_pages(atc, base + 10, >+ (page_size * 20) + (page_size - 10)) >+ == 21); >+ g_assert(atc_get_max_number_of_pages(atc, base + 10, >+ (page_size * 20) + >+ (page_size - 10 + 1)) == 22); >+} >+ >+int main(int argc, char **argv) >+{ >+ g_test_init(&argc, &argv, NULL); >+ g_test_add_func("/atc/test_creation_parameters", >test_creation_parameters); >+ g_test_add_func("/atc/test_single_entry", test_single_entry); >+ g_test_add_func("/atc/test_page_boundaries", test_page_boundaries); >+ g_test_add_func("/atc/test_huge_page", test_huge_page); >+ g_test_add_func("/atc/test_pasid", test_pasid); >+ g_test_add_func("/atc/test_large_address", test_large_address); >+ g_test_add_func("/atc/test_bigger_page", test_bigger_page); >+ g_test_add_func("/atc/test_unknown_pasid", test_unknown_pasid); >+ g_test_add_func("/atc/test_invalidation", test_invalidation); >+ g_test_add_func("/atc/test_delete_address_space_cache", >+ test_delete_address_space_cache); >+ g_test_add_func("/atc/test_invalidate_entire_address_space", >+ test_invalidate_entire_address_space); >+ g_test_add_func("/atc/test_reset", test_reset); >+ g_test_add_func("/atc/test_get_max_number_of_pages", >+ test_get_max_number_of_pages); >+ return g_test_run(); >+} >diff --git a/util/atc.c b/util/atc.c >new file mode 100644 >index 0000000000..d951532e26 >--- /dev/null >+++ b/util/atc.c >@@ -0,0 +1,211 @@ >+/* >+ * QEMU emulation of an ATC >+ * >+ * This program is free software; you can redistribute it and/or modify >+ * it under the terms of the GNU General Public License as published by >+ * the Free Software Foundation; either version 2 of the License, or >+ * (at your option) any later version. >+ >+ * This program is distributed in the hope that it will be useful, >+ * but WITHOUT ANY WARRANTY; without even the implied warranty of >+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >+ * GNU General Public License for more details. >+ >+ * You should have received a copy of the GNU General Public License along >+ * with this program; if not, see <http://www.gnu.org/licenses/>. >+ */ >+ >+#include "util/atc.h" >+ >+ >+#define PAGE_TABLE_ENTRY_SIZE 8 >+ >+/* a pasid is hashed using the identity function */ >+static guint atc_pasid_key_hash(gconstpointer v) >+{ >+ return (guint)(uintptr_t)v; /* pasid */ >+} >+ >+/* pasid equality */ >+static gboolean atc_pasid_key_equal(gconstpointer v1, gconstpointer v2) >+{ >+ return v1 == v2; >+} >+ >+/* Hash function for IOTLB entries */ >+static guint atc_addr_key_hash(gconstpointer v) >+{ >+ hwaddr addr = (hwaddr)v; >+ return (guint)((addr >> 32) ^ (addr & 0xffffffffU)); >+} >+ >+/* Equality test for IOTLB entries */ >+static gboolean atc_addr_key_equal(gconstpointer v1, gconstpointer v2) >+{ >+ return (hwaddr)v1 == (hwaddr)v2; >+} >+ >+static void atc_address_space_free(void *as) >+{ >+ g_hash_table_unref(as); >+} >+ >+/* return log2(val), or UINT8_MAX if val is not a power of 2 */ >+static uint8_t ilog2(uint64_t val) >+{ >+ uint8_t result = 0; >+ while (val != 1) { >+ if (val & 1) { >+ return UINT8_MAX; >+ } >+ >+ val >>= 1; >+ result += 1; >+ } >+ return result; >+} >+ >+ATC *atc_new(uint64_t page_size, uint8_t address_width) >+{ >+ ATC *atc; >+ uint8_t log_page_size = ilog2(page_size); >+ /* number of bits each used to store all the intermediate indexes */ >+ uint64_t addr_lookup_indexes_size; >+ >+ if (log_page_size == UINT8_MAX) { >+ return NULL; >+ } >+ /* >+ * We only support page table entries of 8 (PAGE_TABLE_ENTRY_SIZE) >bytes >+ * log2(page_size / 8) = log2(page_size) - 3 >+ * is the level offset >+ */ >+ if (log_page_size <= 3) { >+ return NULL; >+ } >+ >+ atc = g_new0(ATC, 1); >+ atc->address_spaces = g_hash_table_new_full(atc_pasid_key_hash, >+ atc_pasid_key_equal, >+ NULL, atc_address_space_free); >+ atc->level_offset = log_page_size - 3; >+ /* at this point, we know that page_size is a power of 2 */ >+ atc->min_addr_mask = page_size - 1; >+ addr_lookup_indexes_size = address_width - log_page_size; >+ if ((addr_lookup_indexes_size % atc->level_offset) != 0) { >+ goto error; >+ } >+ atc->levels = addr_lookup_indexes_size / atc->level_offset; >+ atc->page_size = page_size; >+ return atc; >+ >+error: >+ g_free(atc); >+ return NULL; >+} >+ >+static inline GHashTable *atc_get_address_space_cache(ATC *atc, uint32_t >pasid) >+{ >+ return g_hash_table_lookup(atc->address_spaces, >+ (gconstpointer)(uintptr_t)pasid); >+} >+ >+void atc_create_address_space_cache(ATC *atc, uint32_t pasid) >+{ >+ GHashTable *as_cache; >+ >+ as_cache = atc_get_address_space_cache(atc, pasid); >+ if (!as_cache) { >+ as_cache = g_hash_table_new_full(atc_addr_key_hash, >+ atc_addr_key_equal, >+ NULL, g_free); >+ g_hash_table_replace(atc->address_spaces, >+ (gpointer)(uintptr_t)pasid, as_cache); >+ } >+} >+ >+void atc_delete_address_space_cache(ATC *atc, uint32_t pasid) >+{ >+ g_hash_table_remove(atc->address_spaces, (gpointer)(uintptr_t)pasid); >+} >+ >+int atc_update(ATC *atc, IOMMUTLBEntry *entry) >+{ >+ IOMMUTLBEntry *value; >+ GHashTable *as_cache = atc_get_address_space_cache(atc, entry- >>pasid); >+ if (!as_cache) { >+ return -ENODEV; >+ } >+ value = g_memdup2(entry, sizeof(*value)); >+ g_hash_table_replace(as_cache, (gpointer)(entry->iova), value); >+ return 0; >+} >+ >+IOMMUTLBEntry *atc_lookup(ATC *atc, uint32_t pasid, hwaddr addr) >+{ >+ IOMMUTLBEntry *entry; >+ hwaddr mask = atc->min_addr_mask; >+ hwaddr key = addr & (~mask); >+ GHashTable *as_cache = atc_get_address_space_cache(atc, pasid); >+ >+ if (!as_cache) { >+ return NULL; >+ } >+ >+ /* >+ * Iterate over the possible page sizes and try to find a hit >+ */ >+ for (uint8_t level = 0; level < atc->levels; ++level) { >+ entry = g_hash_table_lookup(as_cache, (gconstpointer)key); >+ if (entry) { >+ return entry; >+ } >+ mask = (mask << atc->level_offset) | ((1 << atc->level_offset) - 1); >+ key = addr & (~mask); >+ } >+ >+ return NULL; >+} >+ >+static gboolean atc_invalidate_entry_predicate(gpointer key, gpointer >value, >+ gpointer user_data) >+{ >+ IOMMUTLBEntry *entry = (IOMMUTLBEntry *)value; >+ IOMMUTLBEntry *target = (IOMMUTLBEntry *)user_data; >+ hwaddr target_mask = ~target->addr_mask; >+ hwaddr entry_mask = ~entry->addr_mask; >+ return ((target->iova & target_mask) == (entry->iova & target_mask)) || >+ ((target->iova & entry_mask) == (entry->iova & entry_mask)); >+} >+ >+void atc_invalidate(ATC *atc, IOMMUTLBEntry *entry) >+{ >+ GHashTable *as_cache = atc_get_address_space_cache(atc, entry- >>pasid); >+ if (!as_cache) { >+ return; >+ } >+ g_hash_table_foreach_remove(as_cache, >+ atc_invalidate_entry_predicate, >+ entry); >+} >+ >+void atc_destroy(ATC *atc) >+{ >+ g_hash_table_unref(atc->address_spaces); >+} >+ >+size_t atc_get_max_number_of_pages(ATC *atc, hwaddr addr, size_t >length) >+{ >+ hwaddr page_mask = ~(atc->min_addr_mask); >+ size_t result = (length / atc->page_size); >+ if ((((addr & page_mask) + length - 1) & page_mask) != >+ ((addr + length - 1) & page_mask)) { >+ result += 1; >+ } >+ return result + (length % atc->page_size != 0 ? 1 : 0); >+} >+ >+void atc_reset(ATC *atc) >+{ >+ g_hash_table_remove_all(atc->address_spaces); >+} >diff --git a/util/atc.h b/util/atc.h >new file mode 100644 >index 0000000000..8be95f5cca >--- /dev/null >+++ b/util/atc.h >@@ -0,0 +1,117 @@ >+/* >+ * QEMU emulation of an ATC >+ * >+ * This program is free software; you can redistribute it and/or modify >+ * it under the terms of the GNU General Public License as published by >+ * the Free Software Foundation; either version 2 of the License, or >+ * (at your option) any later version. >+ >+ * This program is distributed in the hope that it will be useful, >+ * but WITHOUT ANY WARRANTY; without even the implied warranty of >+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >+ * GNU General Public License for more details. >+ >+ * You should have received a copy of the GNU General Public License along >+ * with this program; if not, see <http://www.gnu.org/licenses/>. >+ */ >+ >+#ifndef UTIL_ATC_H >+#define UTIL_ATC_H >+ >+#include "qemu/osdep.h" >+#include "exec/memory.h" >+ >+typedef struct ATC { >+ GHashTable *address_spaces; /* Key : pasid, value : GHashTable */ >+ hwaddr min_addr_mask; >+ uint64_t page_size; >+ uint8_t levels; >+ uint8_t level_offset; >+} ATC; >+ >+/* >+ * atc_new: Create an ATC. >+ * >+ * Return an ATC or NULL if the creation failed >+ * >+ * @page_size: #PCIDevice doing the memory access >+ * @address_width: width of the virtual addresses used by the IOMMU (in >bits) >+ */ >+ATC *atc_new(uint64_t page_size, uint8_t address_width); >+ >+/* >+ * atc_update: Insert or update an entry in the cache >+ * >+ * Return 0 if the operation succeeds, a negative error code otherwise >+ * >+ * The insertion will fail if the address space associated with this pasid >+ * has not been created with atc_create_address_space_cache >+ * >+ * @atc: the ATC to update >+ * @entry: the tlb entry to insert into the cache >+ */ >+int atc_update(ATC *atc, IOMMUTLBEntry *entry); >+ >+/* >+ * atc_create_address_space_cache: delare a new address space >+ * identified by a PASID >+ * >+ * @atc: the ATC to update >+ * @pasid: the pasid of the address space to be created >+ */ >+void atc_create_address_space_cache(ATC *atc, uint32_t pasid); >+ >+/* >+ * atc_delete_address_space_cache: delete an address space >+ * identified by a PASID >+ * >+ * @atc: the ATC to update >+ * @pasid: the pasid of the address space to be deleted >+ */ >+void atc_delete_address_space_cache(ATC *atc, uint32_t pasid); >+ >+/* >+ * atc_lookup: query the cache in a given address space >+ * >+ * @atc: the ATC to query >+ * @pasid: the pasid of the address space to query >+ * @addr: the virtual address to translate >+ */ >+IOMMUTLBEntry *atc_lookup(ATC *atc, uint32_t pasid, hwaddr addr); >+ >+/* >+ * atc_invalidate: invalidate an entry in the cache >+ * >+ * @atc: the ATC to update >+ * @entry: the entry to invalidate >+ */ >+void atc_invalidate(ATC *atc, IOMMUTLBEntry *entry); >+ >+/* >+ * atc_destroy: delete an ATC >+ * >+ * @atc: the cache to be deleted >+ */ >+void atc_destroy(ATC *atc); >+ >+/* >+ * atc_get_max_number_of_pages: get the number of pages a memory >operation >+ * will access if all the pages concerned have the minimum size. >+ * >+ * This function can be used to determine the size of the result array to be >+ * allocated when issuing an ATS request. >+ * >+ * @atc: the cache >+ * @addr: start address >+ * @length: number of bytes accessed from addr >+ */ >+size_t atc_get_max_number_of_pages(ATC *atc, hwaddr addr, size_t >length); >+ >+/* >+ * atc_reset: invalidates all the entries stored in the ATC >+ * >+ * @atc: the cache >+ */ >+void atc_reset(ATC *atc); >+ >+#endif >diff --git a/util/meson.build b/util/meson.build >index 0ef9886be0..a2e0e9e5d7 100644 >--- a/util/meson.build >+++ b/util/meson.build >@@ -94,6 +94,7 @@ if have_block > util_ss.add(files('hbitmap.c')) > util_ss.add(files('hexdump.c')) > util_ss.add(files('iova-tree.c')) >+ util_ss.add(files('atc.c')) > util_ss.add(files('iov.c', 'uri.c')) > util_ss.add(files('nvdimm-utils.c')) > util_ss.add(files('block-helpers.c')) >-- >2.44.0
On 17/05/2024 12:44, Duan, Zhenzhong wrote: > Caution: External email. Do not open attachments or click links, unless this email comes from a known sender and you know the content is safe. > > >> -----Original Message----- >> From: CLEMENT MATHIEU--DRIF <clement.mathieu--drif@eviden.com> >> Subject: [PATCH ats_vtd v2 21/25] atc: generic ATC that can be used by PCIe >> devices that support SVM >> >> As the SVM-capable devices will need to cache translations, we provide >> an first implementation. >> >> This cache uses a two-level design based on hash tables. >> The first level is indexed by a PASID and the second by a virtual addresse. >> >> Signed-off-by: Clément Mathieu--Drif <clement.mathieu--drif@eviden.com> >> --- >> tests/unit/meson.build | 1 + >> tests/unit/test-atc.c | 502 >> +++++++++++++++++++++++++++++++++++++++++ >> util/atc.c | 211 +++++++++++++++++ >> util/atc.h | 117 ++++++++++ >> util/meson.build | 1 + >> 5 files changed, 832 insertions(+) >> create mode 100644 tests/unit/test-atc.c >> create mode 100644 util/atc.c >> create mode 100644 util/atc.h > Maybe the unit test can be split from functional change? will do! >> diff --git a/tests/unit/meson.build b/tests/unit/meson.build >> index 228a21d03c..5c9a6fe9f4 100644 >> --- a/tests/unit/meson.build >> +++ b/tests/unit/meson.build >> @@ -52,6 +52,7 @@ tests = { >> 'test-interval-tree': [], >> 'test-xs-node': [qom], >> 'test-virtio-dmabuf': [meson.project_source_root() / 'hw/display/virtio- >> dmabuf.c'], >> + 'test-atc': [] >> } >> >> if have_system or have_tools >> diff --git a/tests/unit/test-atc.c b/tests/unit/test-atc.c >> new file mode 100644 >> index 0000000000..60fa60924a >> --- /dev/null >> +++ b/tests/unit/test-atc.c >> @@ -0,0 +1,502 @@ >> +/* >> + * This program is free software; you can redistribute it and/or modify >> + * it under the terms of the GNU General Public License as published by >> + * the Free Software Foundation; either version 2 of the License, or >> + * (at your option) any later version. >> + >> + * This program is distributed in the hope that it will be useful, >> + * but WITHOUT ANY WARRANTY; without even the implied warranty of >> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >> + * GNU General Public License for more details. >> + >> + * You should have received a copy of the GNU General Public License along >> + * with this program; if not, see <http://www.gnu.org/licenses/>. >> + */ >> + >> +#include "util/atc.h" >> + >> +static inline bool tlb_entry_equal(IOMMUTLBEntry *e1, IOMMUTLBEntry >> *e2) >> +{ >> + if (!e1 || !e2) { >> + return !e1 && !e2; >> + } >> + return e1->iova == e2->iova && >> + e1->addr_mask == e2->addr_mask && >> + e1->pasid == e2->pasid && >> + e1->perm == e2->perm && >> + e1->target_as == e2->target_as && >> + e1->translated_addr == e2->translated_addr; >> +} >> + >> +static void assert_lookup_equals(ATC *atc, IOMMUTLBEntry *target, >> + uint32_t pasid, hwaddr iova) >> +{ >> + IOMMUTLBEntry *result; >> + result = atc_lookup(atc, pasid, iova); >> + g_assert(tlb_entry_equal(result, target)); >> +} >> + >> +static void check_creation(uint64_t page_size, uint8_t address_width, >> + uint8_t levels, uint8_t level_offset, >> + bool should_work) { >> + ATC *atc = atc_new(page_size, address_width); >> + if (atc) { >> + if (atc->levels != levels || atc->level_offset != level_offset) { >> + g_assert(false); /* ATC created but invalid configuration : fail */ >> + } >> + atc_destroy(atc); >> + g_assert(should_work); >> + } else { >> + g_assert(!should_work); >> + } >> +} >> + >> +static void test_creation_parameters(void) >> +{ >> + check_creation(8, 39, 3, 9, false); >> + check_creation(4095, 39, 3, 9, false); >> + check_creation(4097, 39, 3, 9, false); >> + check_creation(8192, 48, 0, 0, false); >> + >> + check_creation(4096, 38, 0, 0, false); >> + check_creation(4096, 39, 3, 9, true); >> + check_creation(4096, 40, 0, 0, false); >> + check_creation(4096, 47, 0, 0, false); >> + check_creation(4096, 48, 4, 9, true); >> + check_creation(4096, 49, 0, 0, false); >> + check_creation(4096, 56, 0, 0, false); >> + check_creation(4096, 57, 5, 9, true); >> + check_creation(4096, 58, 0, 0, false); >> + >> + check_creation(16384, 35, 0, 0, false); >> + check_creation(16384, 36, 2, 11, true); >> + check_creation(16384, 37, 0, 0, false); >> + check_creation(16384, 46, 0, 0, false); >> + check_creation(16384, 47, 3, 11, true); >> + check_creation(16384, 48, 0, 0, false); >> + check_creation(16384, 57, 0, 0, false); >> + check_creation(16384, 58, 4, 11, true); >> + check_creation(16384, 59, 0, 0, false); >> +} >> + >> +static void test_single_entry(void) >> +{ >> + IOMMUTLBEntry entry = { >> + .iova = 0x123456789000ULL, >> + .addr_mask = 0xfffULL, >> + .pasid = 5, >> + .perm = IOMMU_RW, >> + .translated_addr = 0xdeadbeefULL, >> + }; >> + >> + ATC *atc = atc_new(4096, 48); >> + g_assert(atc); >> + >> + assert_lookup_equals(atc, NULL, entry.pasid, >> + entry.iova + (entry.addr_mask / 2)); >> + >> + atc_create_address_space_cache(atc, entry.pasid); >> + g_assert(atc_update(atc, &entry) == 0); >> + >> + assert_lookup_equals(atc, NULL, entry.pasid + 1, >> + entry.iova + (entry.addr_mask / 2)); >> + assert_lookup_equals(atc, &entry, entry.pasid, >> + entry.iova + (entry.addr_mask / 2)); >> + >> + atc_destroy(atc); >> +} >> + >> +static void test_page_boundaries(void) >> +{ >> + static const uint32_t pasid = 5; >> + static const hwaddr page_size = 4096; >> + >> + /* 2 consecutive entries */ >> + IOMMUTLBEntry e1 = { >> + .iova = 0x123456789000ULL, >> + .addr_mask = page_size - 1, >> + .pasid = pasid, >> + .perm = IOMMU_RW, >> + .translated_addr = 0xdeadbeefULL, >> + }; >> + IOMMUTLBEntry e2 = { >> + .iova = e1.iova + page_size, >> + .addr_mask = page_size - 1, >> + .pasid = pasid, >> + .perm = IOMMU_RW, >> + .translated_addr = 0x900df00dULL, >> + }; >> + >> + ATC *atc = atc_new(page_size, 48); >> + >> + atc_create_address_space_cache(atc, e1.pasid); >> + /* creating the address space twice should not be a problem */ >> + atc_create_address_space_cache(atc, e1.pasid); >> + >> + atc_update(atc, &e1); >> + atc_update(atc, &e2); >> + >> + assert_lookup_equals(atc, NULL, e1.pasid, e1.iova - 1); >> + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >> + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova + e1.addr_mask); >> + g_assert((e1.iova + e1.addr_mask + 1) == e2.iova); >> + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >> + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova + e2.addr_mask); >> + assert_lookup_equals(atc, NULL, e2.pasid, e2.iova + e2.addr_mask + 1); >> + >> + assert_lookup_equals(atc, NULL, e1.pasid + 10, e1.iova); >> + assert_lookup_equals(atc, NULL, e2.pasid + 10, e2.iova); >> + atc_destroy(atc); >> +} >> + >> +static void test_huge_page(void) >> +{ >> + static const uint32_t pasid = 5; >> + static const hwaddr page_size = 4096; >> + IOMMUTLBEntry e1 = { >> + .iova = 0x123456600000ULL, >> + .addr_mask = 0x1fffffULL, >> + .pasid = pasid, >> + .perm = IOMMU_RW, >> + .translated_addr = 0xdeadbeefULL, >> + }; >> + hwaddr addr; >> + >> + ATC *atc = atc_new(page_size, 48); >> + >> + atc_create_address_space_cache(atc, e1.pasid); >> + atc_update(atc, &e1); >> + >> + for (addr = e1.iova; addr <= e1.iova + e1.addr_mask; addr += page_size) { >> + assert_lookup_equals(atc, &e1, e1.pasid, addr); >> + } >> + /* addr is now out of the huge page */ >> + assert_lookup_equals(atc, NULL, e1.pasid, addr); >> + atc_destroy(atc); >> +} >> + >> +static void test_pasid(void) >> +{ >> + hwaddr addr = 0xaaaaaaaaa000ULL; >> + IOMMUTLBEntry e1 = { >> + .iova = addr, >> + .addr_mask = 0xfffULL, >> + .pasid = 8, >> + .perm = IOMMU_RW, >> + .translated_addr = 0xdeadbeefULL, >> + }; >> + IOMMUTLBEntry e2 = { >> + .iova = addr, >> + .addr_mask = 0xfffULL, >> + .pasid = 2, >> + .perm = IOMMU_RW, >> + .translated_addr = 0xb001ULL, >> + }; >> + uint16_t i; >> + >> + ATC *atc = atc_new(4096, 48); >> + >> + atc_create_address_space_cache(atc, e1.pasid); >> + atc_create_address_space_cache(atc, e2.pasid); >> + atc_update(atc, &e1); >> + atc_update(atc, &e2); >> + >> + for (i = 0; i <= MAX(e1.pasid, e2.pasid) + 1; ++i) { >> + if (i == e1.pasid || i == e2.pasid) { >> + continue; >> + } >> + assert_lookup_equals(atc, NULL, i, addr); >> + } >> + assert_lookup_equals(atc, &e1, e1.pasid, addr); >> + assert_lookup_equals(atc, &e1, e1.pasid, addr); >> + atc_destroy(atc); >> +} >> + >> +static void test_large_address(void) >> +{ >> + IOMMUTLBEntry e1 = { >> + .iova = 0xaaaaaaaaa000ULL, >> + .addr_mask = 0xfffULL, >> + .pasid = 8, >> + .perm = IOMMU_RW, >> + .translated_addr = 0x5eeeeeedULL, >> + }; >> + IOMMUTLBEntry e2 = { >> + .iova = 0x1f00baaaaabf000ULL, >> + .addr_mask = 0xfffULL, >> + .pasid = e1.pasid, >> + .perm = IOMMU_RW, >> + .translated_addr = 0xdeadbeefULL, >> + }; >> + >> + ATC *atc = atc_new(4096, 57); >> + >> + atc_create_address_space_cache(atc, e1.pasid); >> + atc_update(atc, &e1); >> + atc_update(atc, &e2); >> + >> + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >> + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >> + atc_destroy(atc); >> +} >> + >> +static void test_bigger_page(void) >> +{ >> + IOMMUTLBEntry e1 = { >> + .iova = 0xaabbccdde000ULL, >> + .addr_mask = 0x1fffULL, >> + .pasid = 1, >> + .perm = IOMMU_RW, >> + .translated_addr = 0x5eeeeeedULL, >> + }; >> + hwaddr i; >> + >> + ATC *atc = atc_new(8192, 43); >> + >> + atc_create_address_space_cache(atc, e1.pasid); >> + atc_update(atc, &e1); >> + >> + i = e1.iova & (~e1.addr_mask); >> + assert_lookup_equals(atc, NULL, e1.pasid, i - 1); >> + while (i <= e1.iova + e1.addr_mask) { >> + assert_lookup_equals(atc, &e1, e1.pasid, i); >> + ++i; >> + } >> + assert_lookup_equals(atc, NULL, e1.pasid, i); >> + atc_destroy(atc); >> +} >> + >> +static void test_unknown_pasid(void) >> +{ >> + IOMMUTLBEntry e1 = { >> + .iova = 0xaabbccfff000ULL, >> + .addr_mask = 0xfffULL, >> + .pasid = 1, >> + .perm = IOMMU_RW, >> + .translated_addr = 0x5eeeeeedULL, >> + }; >> + >> + ATC *atc = atc_new(4096, 48); >> + g_assert(atc_update(atc, &e1) != 0); >> + assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); >> +} >> + >> +static void test_invalidation(void) >> +{ >> + static uint64_t page_size = 4096; >> + IOMMUTLBEntry e1 = { >> + .iova = 0xaabbccddf000ULL, >> + .addr_mask = 0xfffULL, >> + .pasid = 1, >> + .perm = IOMMU_RW, >> + .translated_addr = 0x5eeeeeedULL, >> + }; >> + IOMMUTLBEntry e2 = { >> + .iova = 0xffe00000ULL, >> + .addr_mask = 0x1fffffULL, >> + .pasid = 1, >> + .perm = IOMMU_RW, >> + .translated_addr = 0xb000001ULL, >> + }; >> + IOMMUTLBEntry e3; >> + >> + ATC *atc = atc_new(page_size , 48); >> + atc_create_address_space_cache(atc, e1.pasid); >> + >> + atc_update(atc, &e1); >> + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >> + atc_invalidate(atc, &e1); >> + assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); >> + >> + atc_update(atc, &e1); >> + atc_update(atc, &e2); >> + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >> + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >> + atc_invalidate(atc, &e2); >> + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >> + assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); >> + >> + /* invalidate a huge page by invalidating a small region */ >> + for (hwaddr addr = e2.iova; addr <= (e2.iova + e2.addr_mask); >> + addr += page_size) { >> + atc_update(atc, &e2); >> + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >> + e3 = (IOMMUTLBEntry){ >> + .iova = addr, >> + .addr_mask = page_size - 1, >> + .pasid = e2.pasid, >> + .perm = IOMMU_RW, >> + .translated_addr = 0, >> + }; >> + atc_invalidate(atc, &e3); >> + assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); >> + } >> +} >> + >> +static void test_delete_address_space_cache(void) >> +{ >> + static uint64_t page_size = 4096; >> + IOMMUTLBEntry e1 = { >> + .iova = 0xaabbccddf000ULL, >> + .addr_mask = 0xfffULL, >> + .pasid = 1, >> + .perm = IOMMU_RW, >> + .translated_addr = 0x5eeeeeedULL, >> + }; >> + IOMMUTLBEntry e2 = { >> + .iova = e1.iova, >> + .addr_mask = 0xfffULL, >> + .pasid = 2, >> + .perm = IOMMU_RW, >> + .translated_addr = 0x5eeeeeedULL, >> + }; >> + >> + ATC *atc = atc_new(page_size , 48); >> + atc_create_address_space_cache(atc, e1.pasid); >> + >> + atc_update(atc, &e1); >> + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >> + atc_invalidate(atc, &e2); /* unkown pasid : is a nop*/ >> + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >> + >> + atc_create_address_space_cache(atc, e2.pasid); >> + atc_update(atc, &e2); >> + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >> + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >> + atc_invalidate(atc, &e1); >> + /* e1 has been removed but e2 is still there */ >> + assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); >> + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >> + >> + atc_update(atc, &e1); >> + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >> + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >> + >> + atc_delete_address_space_cache(atc, e2.pasid); >> + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >> + assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); >> +} >> + >> +static void test_invalidate_entire_address_space(void) >> +{ >> + static uint64_t page_size = 4096; >> + IOMMUTLBEntry e1 = { >> + .iova = 0x1000ULL, >> + .addr_mask = 0xfffULL, >> + .pasid = 1, >> + .perm = IOMMU_RW, >> + .translated_addr = 0x5eedULL, >> + }; >> + IOMMUTLBEntry e2 = { >> + .iova = 0xfffffffff000ULL, >> + .addr_mask = 0xfffULL, >> + .pasid = 1, >> + .perm = IOMMU_RW, >> + .translated_addr = 0xbeefULL, >> + }; >> + IOMMUTLBEntry e3 = { >> + .iova = 0, >> + .addr_mask = 0xffffffffffffffffULL, >> + .pasid = 1, >> + .perm = IOMMU_RW, >> + .translated_addr = 0, >> + }; >> + >> + ATC *atc = atc_new(page_size , 48); >> + atc_create_address_space_cache(atc, e1.pasid); >> + >> + atc_update(atc, &e1); >> + atc_update(atc, &e2); >> + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >> + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >> + atc_invalidate(atc, &e3); >> + /* e1 has been removed but e2 is still there */ >> + assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); >> + assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); >> + >> + atc_destroy(atc); >> +} >> + >> +static void test_reset(void) >> +{ >> + static uint64_t page_size = 4096; >> + IOMMUTLBEntry e1 = { >> + .iova = 0x1000ULL, >> + .addr_mask = 0xfffULL, >> + .pasid = 1, >> + .perm = IOMMU_RW, >> + .translated_addr = 0x5eedULL, >> + }; >> + IOMMUTLBEntry e2 = { >> + .iova = 0xfffffffff000ULL, >> + .addr_mask = 0xfffULL, >> + .pasid = 2, >> + .perm = IOMMU_RW, >> + .translated_addr = 0xbeefULL, >> + }; >> + >> + ATC *atc = atc_new(page_size , 48); >> + atc_create_address_space_cache(atc, e1.pasid); >> + atc_create_address_space_cache(atc, e2.pasid); >> + atc_update(atc, &e1); >> + atc_update(atc, &e2); >> + >> + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); >> + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); >> + >> + atc_reset(atc); >> + >> + assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); >> + assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); >> +} >> + >> +static void test_get_max_number_of_pages(void) >> +{ >> + static uint64_t page_size = 4096; >> + hwaddr base = 0xc0fee000; /* aligned */ >> + ATC *atc = atc_new(page_size , 48); >> + g_assert(atc_get_max_number_of_pages(atc, base, page_size / 2) == 1); >> + g_assert(atc_get_max_number_of_pages(atc, base, page_size) == 1); >> + g_assert(atc_get_max_number_of_pages(atc, base, page_size + 1) == 2); >> + >> + g_assert(atc_get_max_number_of_pages(atc, base + 10, 1) == 1); >> + g_assert(atc_get_max_number_of_pages(atc, base + 10, page_size - 10) >> == 1); >> + g_assert(atc_get_max_number_of_pages(atc, base + 10, >> + page_size - 10 + 1) == 2); >> + g_assert(atc_get_max_number_of_pages(atc, base + 10, >> + page_size - 10 + 2) == 2); >> + >> + g_assert(atc_get_max_number_of_pages(atc, base + page_size - 1, 1) == >> 1); >> + g_assert(atc_get_max_number_of_pages(atc, base + page_size - 1, 2) == >> 2); >> + g_assert(atc_get_max_number_of_pages(atc, base + page_size - 1, 3) == >> 2); >> + >> + g_assert(atc_get_max_number_of_pages(atc, base + 10, page_size * 20) >> == 21); >> + g_assert(atc_get_max_number_of_pages(atc, base + 10, >> + (page_size * 20) + (page_size - 10)) >> + == 21); >> + g_assert(atc_get_max_number_of_pages(atc, base + 10, >> + (page_size * 20) + >> + (page_size - 10 + 1)) == 22); >> +} >> + >> +int main(int argc, char **argv) >> +{ >> + g_test_init(&argc, &argv, NULL); >> + g_test_add_func("/atc/test_creation_parameters", >> test_creation_parameters); >> + g_test_add_func("/atc/test_single_entry", test_single_entry); >> + g_test_add_func("/atc/test_page_boundaries", test_page_boundaries); >> + g_test_add_func("/atc/test_huge_page", test_huge_page); >> + g_test_add_func("/atc/test_pasid", test_pasid); >> + g_test_add_func("/atc/test_large_address", test_large_address); >> + g_test_add_func("/atc/test_bigger_page", test_bigger_page); >> + g_test_add_func("/atc/test_unknown_pasid", test_unknown_pasid); >> + g_test_add_func("/atc/test_invalidation", test_invalidation); >> + g_test_add_func("/atc/test_delete_address_space_cache", >> + test_delete_address_space_cache); >> + g_test_add_func("/atc/test_invalidate_entire_address_space", >> + test_invalidate_entire_address_space); >> + g_test_add_func("/atc/test_reset", test_reset); >> + g_test_add_func("/atc/test_get_max_number_of_pages", >> + test_get_max_number_of_pages); >> + return g_test_run(); >> +} >> diff --git a/util/atc.c b/util/atc.c >> new file mode 100644 >> index 0000000000..d951532e26 >> --- /dev/null >> +++ b/util/atc.c >> @@ -0,0 +1,211 @@ >> +/* >> + * QEMU emulation of an ATC >> + * >> + * This program is free software; you can redistribute it and/or modify >> + * it under the terms of the GNU General Public License as published by >> + * the Free Software Foundation; either version 2 of the License, or >> + * (at your option) any later version. >> + >> + * This program is distributed in the hope that it will be useful, >> + * but WITHOUT ANY WARRANTY; without even the implied warranty of >> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >> + * GNU General Public License for more details. >> + >> + * You should have received a copy of the GNU General Public License along >> + * with this program; if not, see <http://www.gnu.org/licenses/>. >> + */ >> + >> +#include "util/atc.h" >> + >> + >> +#define PAGE_TABLE_ENTRY_SIZE 8 >> + >> +/* a pasid is hashed using the identity function */ >> +static guint atc_pasid_key_hash(gconstpointer v) >> +{ >> + return (guint)(uintptr_t)v; /* pasid */ >> +} >> + >> +/* pasid equality */ >> +static gboolean atc_pasid_key_equal(gconstpointer v1, gconstpointer v2) >> +{ >> + return v1 == v2; >> +} >> + >> +/* Hash function for IOTLB entries */ >> +static guint atc_addr_key_hash(gconstpointer v) >> +{ >> + hwaddr addr = (hwaddr)v; >> + return (guint)((addr >> 32) ^ (addr & 0xffffffffU)); >> +} >> + >> +/* Equality test for IOTLB entries */ >> +static gboolean atc_addr_key_equal(gconstpointer v1, gconstpointer v2) >> +{ >> + return (hwaddr)v1 == (hwaddr)v2; >> +} >> + >> +static void atc_address_space_free(void *as) >> +{ >> + g_hash_table_unref(as); >> +} >> + >> +/* return log2(val), or UINT8_MAX if val is not a power of 2 */ >> +static uint8_t ilog2(uint64_t val) >> +{ >> + uint8_t result = 0; >> + while (val != 1) { >> + if (val & 1) { >> + return UINT8_MAX; >> + } >> + >> + val >>= 1; >> + result += 1; >> + } >> + return result; >> +} >> + >> +ATC *atc_new(uint64_t page_size, uint8_t address_width) >> +{ >> + ATC *atc; >> + uint8_t log_page_size = ilog2(page_size); >> + /* number of bits each used to store all the intermediate indexes */ >> + uint64_t addr_lookup_indexes_size; >> + >> + if (log_page_size == UINT8_MAX) { >> + return NULL; >> + } >> + /* >> + * We only support page table entries of 8 (PAGE_TABLE_ENTRY_SIZE) >> bytes >> + * log2(page_size / 8) = log2(page_size) - 3 >> + * is the level offset >> + */ >> + if (log_page_size <= 3) { >> + return NULL; >> + } >> + >> + atc = g_new0(ATC, 1); >> + atc->address_spaces = g_hash_table_new_full(atc_pasid_key_hash, >> + atc_pasid_key_equal, >> + NULL, atc_address_space_free); >> + atc->level_offset = log_page_size - 3; >> + /* at this point, we know that page_size is a power of 2 */ >> + atc->min_addr_mask = page_size - 1; >> + addr_lookup_indexes_size = address_width - log_page_size; >> + if ((addr_lookup_indexes_size % atc->level_offset) != 0) { >> + goto error; >> + } >> + atc->levels = addr_lookup_indexes_size / atc->level_offset; >> + atc->page_size = page_size; >> + return atc; >> + >> +error: >> + g_free(atc); >> + return NULL; >> +} >> + >> +static inline GHashTable *atc_get_address_space_cache(ATC *atc, uint32_t >> pasid) >> +{ >> + return g_hash_table_lookup(atc->address_spaces, >> + (gconstpointer)(uintptr_t)pasid); >> +} >> + >> +void atc_create_address_space_cache(ATC *atc, uint32_t pasid) >> +{ >> + GHashTable *as_cache; >> + >> + as_cache = atc_get_address_space_cache(atc, pasid); >> + if (!as_cache) { >> + as_cache = g_hash_table_new_full(atc_addr_key_hash, >> + atc_addr_key_equal, >> + NULL, g_free); >> + g_hash_table_replace(atc->address_spaces, >> + (gpointer)(uintptr_t)pasid, as_cache); >> + } >> +} >> + >> +void atc_delete_address_space_cache(ATC *atc, uint32_t pasid) >> +{ >> + g_hash_table_remove(atc->address_spaces, (gpointer)(uintptr_t)pasid); >> +} >> + >> +int atc_update(ATC *atc, IOMMUTLBEntry *entry) >> +{ >> + IOMMUTLBEntry *value; >> + GHashTable *as_cache = atc_get_address_space_cache(atc, entry- >>> pasid); >> + if (!as_cache) { >> + return -ENODEV; >> + } >> + value = g_memdup2(entry, sizeof(*value)); >> + g_hash_table_replace(as_cache, (gpointer)(entry->iova), value); >> + return 0; >> +} >> + >> +IOMMUTLBEntry *atc_lookup(ATC *atc, uint32_t pasid, hwaddr addr) >> +{ >> + IOMMUTLBEntry *entry; >> + hwaddr mask = atc->min_addr_mask; >> + hwaddr key = addr & (~mask); >> + GHashTable *as_cache = atc_get_address_space_cache(atc, pasid); >> + >> + if (!as_cache) { >> + return NULL; >> + } >> + >> + /* >> + * Iterate over the possible page sizes and try to find a hit >> + */ >> + for (uint8_t level = 0; level < atc->levels; ++level) { >> + entry = g_hash_table_lookup(as_cache, (gconstpointer)key); >> + if (entry) { >> + return entry; >> + } >> + mask = (mask << atc->level_offset) | ((1 << atc->level_offset) - 1); >> + key = addr & (~mask); >> + } >> + >> + return NULL; >> +} >> + >> +static gboolean atc_invalidate_entry_predicate(gpointer key, gpointer >> value, >> + gpointer user_data) >> +{ >> + IOMMUTLBEntry *entry = (IOMMUTLBEntry *)value; >> + IOMMUTLBEntry *target = (IOMMUTLBEntry *)user_data; >> + hwaddr target_mask = ~target->addr_mask; >> + hwaddr entry_mask = ~entry->addr_mask; >> + return ((target->iova & target_mask) == (entry->iova & target_mask)) || >> + ((target->iova & entry_mask) == (entry->iova & entry_mask)); >> +} >> + >> +void atc_invalidate(ATC *atc, IOMMUTLBEntry *entry) >> +{ >> + GHashTable *as_cache = atc_get_address_space_cache(atc, entry- >>> pasid); >> + if (!as_cache) { >> + return; >> + } >> + g_hash_table_foreach_remove(as_cache, >> + atc_invalidate_entry_predicate, >> + entry); >> +} >> + >> +void atc_destroy(ATC *atc) >> +{ >> + g_hash_table_unref(atc->address_spaces); >> +} >> + >> +size_t atc_get_max_number_of_pages(ATC *atc, hwaddr addr, size_t >> length) >> +{ >> + hwaddr page_mask = ~(atc->min_addr_mask); >> + size_t result = (length / atc->page_size); >> + if ((((addr & page_mask) + length - 1) & page_mask) != >> + ((addr + length - 1) & page_mask)) { >> + result += 1; >> + } >> + return result + (length % atc->page_size != 0 ? 1 : 0); >> +} >> + >> +void atc_reset(ATC *atc) >> +{ >> + g_hash_table_remove_all(atc->address_spaces); >> +} >> diff --git a/util/atc.h b/util/atc.h >> new file mode 100644 >> index 0000000000..8be95f5cca >> --- /dev/null >> +++ b/util/atc.h >> @@ -0,0 +1,117 @@ >> +/* >> + * QEMU emulation of an ATC >> + * >> + * This program is free software; you can redistribute it and/or modify >> + * it under the terms of the GNU General Public License as published by >> + * the Free Software Foundation; either version 2 of the License, or >> + * (at your option) any later version. >> + >> + * This program is distributed in the hope that it will be useful, >> + * but WITHOUT ANY WARRANTY; without even the implied warranty of >> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >> + * GNU General Public License for more details. >> + >> + * You should have received a copy of the GNU General Public License along >> + * with this program; if not, see <http://www.gnu.org/licenses/>. >> + */ >> + >> +#ifndef UTIL_ATC_H >> +#define UTIL_ATC_H >> + >> +#include "qemu/osdep.h" >> +#include "exec/memory.h" >> + >> +typedef struct ATC { >> + GHashTable *address_spaces; /* Key : pasid, value : GHashTable */ >> + hwaddr min_addr_mask; >> + uint64_t page_size; >> + uint8_t levels; >> + uint8_t level_offset; >> +} ATC; >> + >> +/* >> + * atc_new: Create an ATC. >> + * >> + * Return an ATC or NULL if the creation failed >> + * >> + * @page_size: #PCIDevice doing the memory access >> + * @address_width: width of the virtual addresses used by the IOMMU (in >> bits) >> + */ >> +ATC *atc_new(uint64_t page_size, uint8_t address_width); >> + >> +/* >> + * atc_update: Insert or update an entry in the cache >> + * >> + * Return 0 if the operation succeeds, a negative error code otherwise >> + * >> + * The insertion will fail if the address space associated with this pasid >> + * has not been created with atc_create_address_space_cache >> + * >> + * @atc: the ATC to update >> + * @entry: the tlb entry to insert into the cache >> + */ >> +int atc_update(ATC *atc, IOMMUTLBEntry *entry); >> + >> +/* >> + * atc_create_address_space_cache: delare a new address space >> + * identified by a PASID >> + * >> + * @atc: the ATC to update >> + * @pasid: the pasid of the address space to be created >> + */ >> +void atc_create_address_space_cache(ATC *atc, uint32_t pasid); >> + >> +/* >> + * atc_delete_address_space_cache: delete an address space >> + * identified by a PASID >> + * >> + * @atc: the ATC to update >> + * @pasid: the pasid of the address space to be deleted >> + */ >> +void atc_delete_address_space_cache(ATC *atc, uint32_t pasid); >> + >> +/* >> + * atc_lookup: query the cache in a given address space >> + * >> + * @atc: the ATC to query >> + * @pasid: the pasid of the address space to query >> + * @addr: the virtual address to translate >> + */ >> +IOMMUTLBEntry *atc_lookup(ATC *atc, uint32_t pasid, hwaddr addr); >> + >> +/* >> + * atc_invalidate: invalidate an entry in the cache >> + * >> + * @atc: the ATC to update >> + * @entry: the entry to invalidate >> + */ >> +void atc_invalidate(ATC *atc, IOMMUTLBEntry *entry); >> + >> +/* >> + * atc_destroy: delete an ATC >> + * >> + * @atc: the cache to be deleted >> + */ >> +void atc_destroy(ATC *atc); >> + >> +/* >> + * atc_get_max_number_of_pages: get the number of pages a memory >> operation >> + * will access if all the pages concerned have the minimum size. >> + * >> + * This function can be used to determine the size of the result array to be >> + * allocated when issuing an ATS request. >> + * >> + * @atc: the cache >> + * @addr: start address >> + * @length: number of bytes accessed from addr >> + */ >> +size_t atc_get_max_number_of_pages(ATC *atc, hwaddr addr, size_t >> length); >> + >> +/* >> + * atc_reset: invalidates all the entries stored in the ATC >> + * >> + * @atc: the cache >> + */ >> +void atc_reset(ATC *atc); >> + >> +#endif >> diff --git a/util/meson.build b/util/meson.build >> index 0ef9886be0..a2e0e9e5d7 100644 >> --- a/util/meson.build >> +++ b/util/meson.build >> @@ -94,6 +94,7 @@ if have_block >> util_ss.add(files('hbitmap.c')) >> util_ss.add(files('hexdump.c')) >> util_ss.add(files('iova-tree.c')) >> + util_ss.add(files('atc.c')) >> util_ss.add(files('iov.c', 'uri.c')) >> util_ss.add(files('nvdimm-utils.c')) >> util_ss.add(files('block-helpers.c')) >> -- >> 2.44.0
diff --git a/tests/unit/meson.build b/tests/unit/meson.build index 228a21d03c..5c9a6fe9f4 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -52,6 +52,7 @@ tests = { 'test-interval-tree': [], 'test-xs-node': [qom], 'test-virtio-dmabuf': [meson.project_source_root() / 'hw/display/virtio-dmabuf.c'], + 'test-atc': [] } if have_system or have_tools diff --git a/tests/unit/test-atc.c b/tests/unit/test-atc.c new file mode 100644 index 0000000000..60fa60924a --- /dev/null +++ b/tests/unit/test-atc.c @@ -0,0 +1,502 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License along + * with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#include "util/atc.h" + +static inline bool tlb_entry_equal(IOMMUTLBEntry *e1, IOMMUTLBEntry *e2) +{ + if (!e1 || !e2) { + return !e1 && !e2; + } + return e1->iova == e2->iova && + e1->addr_mask == e2->addr_mask && + e1->pasid == e2->pasid && + e1->perm == e2->perm && + e1->target_as == e2->target_as && + e1->translated_addr == e2->translated_addr; +} + +static void assert_lookup_equals(ATC *atc, IOMMUTLBEntry *target, + uint32_t pasid, hwaddr iova) +{ + IOMMUTLBEntry *result; + result = atc_lookup(atc, pasid, iova); + g_assert(tlb_entry_equal(result, target)); +} + +static void check_creation(uint64_t page_size, uint8_t address_width, + uint8_t levels, uint8_t level_offset, + bool should_work) { + ATC *atc = atc_new(page_size, address_width); + if (atc) { + if (atc->levels != levels || atc->level_offset != level_offset) { + g_assert(false); /* ATC created but invalid configuration : fail */ + } + atc_destroy(atc); + g_assert(should_work); + } else { + g_assert(!should_work); + } +} + +static void test_creation_parameters(void) +{ + check_creation(8, 39, 3, 9, false); + check_creation(4095, 39, 3, 9, false); + check_creation(4097, 39, 3, 9, false); + check_creation(8192, 48, 0, 0, false); + + check_creation(4096, 38, 0, 0, false); + check_creation(4096, 39, 3, 9, true); + check_creation(4096, 40, 0, 0, false); + check_creation(4096, 47, 0, 0, false); + check_creation(4096, 48, 4, 9, true); + check_creation(4096, 49, 0, 0, false); + check_creation(4096, 56, 0, 0, false); + check_creation(4096, 57, 5, 9, true); + check_creation(4096, 58, 0, 0, false); + + check_creation(16384, 35, 0, 0, false); + check_creation(16384, 36, 2, 11, true); + check_creation(16384, 37, 0, 0, false); + check_creation(16384, 46, 0, 0, false); + check_creation(16384, 47, 3, 11, true); + check_creation(16384, 48, 0, 0, false); + check_creation(16384, 57, 0, 0, false); + check_creation(16384, 58, 4, 11, true); + check_creation(16384, 59, 0, 0, false); +} + +static void test_single_entry(void) +{ + IOMMUTLBEntry entry = { + .iova = 0x123456789000ULL, + .addr_mask = 0xfffULL, + .pasid = 5, + .perm = IOMMU_RW, + .translated_addr = 0xdeadbeefULL, + }; + + ATC *atc = atc_new(4096, 48); + g_assert(atc); + + assert_lookup_equals(atc, NULL, entry.pasid, + entry.iova + (entry.addr_mask / 2)); + + atc_create_address_space_cache(atc, entry.pasid); + g_assert(atc_update(atc, &entry) == 0); + + assert_lookup_equals(atc, NULL, entry.pasid + 1, + entry.iova + (entry.addr_mask / 2)); + assert_lookup_equals(atc, &entry, entry.pasid, + entry.iova + (entry.addr_mask / 2)); + + atc_destroy(atc); +} + +static void test_page_boundaries(void) +{ + static const uint32_t pasid = 5; + static const hwaddr page_size = 4096; + + /* 2 consecutive entries */ + IOMMUTLBEntry e1 = { + .iova = 0x123456789000ULL, + .addr_mask = page_size - 1, + .pasid = pasid, + .perm = IOMMU_RW, + .translated_addr = 0xdeadbeefULL, + }; + IOMMUTLBEntry e2 = { + .iova = e1.iova + page_size, + .addr_mask = page_size - 1, + .pasid = pasid, + .perm = IOMMU_RW, + .translated_addr = 0x900df00dULL, + }; + + ATC *atc = atc_new(page_size, 48); + + atc_create_address_space_cache(atc, e1.pasid); + /* creating the address space twice should not be a problem */ + atc_create_address_space_cache(atc, e1.pasid); + + atc_update(atc, &e1); + atc_update(atc, &e2); + + assert_lookup_equals(atc, NULL, e1.pasid, e1.iova - 1); + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova + e1.addr_mask); + g_assert((e1.iova + e1.addr_mask + 1) == e2.iova); + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova + e2.addr_mask); + assert_lookup_equals(atc, NULL, e2.pasid, e2.iova + e2.addr_mask + 1); + + assert_lookup_equals(atc, NULL, e1.pasid + 10, e1.iova); + assert_lookup_equals(atc, NULL, e2.pasid + 10, e2.iova); + atc_destroy(atc); +} + +static void test_huge_page(void) +{ + static const uint32_t pasid = 5; + static const hwaddr page_size = 4096; + IOMMUTLBEntry e1 = { + .iova = 0x123456600000ULL, + .addr_mask = 0x1fffffULL, + .pasid = pasid, + .perm = IOMMU_RW, + .translated_addr = 0xdeadbeefULL, + }; + hwaddr addr; + + ATC *atc = atc_new(page_size, 48); + + atc_create_address_space_cache(atc, e1.pasid); + atc_update(atc, &e1); + + for (addr = e1.iova; addr <= e1.iova + e1.addr_mask; addr += page_size) { + assert_lookup_equals(atc, &e1, e1.pasid, addr); + } + /* addr is now out of the huge page */ + assert_lookup_equals(atc, NULL, e1.pasid, addr); + atc_destroy(atc); +} + +static void test_pasid(void) +{ + hwaddr addr = 0xaaaaaaaaa000ULL; + IOMMUTLBEntry e1 = { + .iova = addr, + .addr_mask = 0xfffULL, + .pasid = 8, + .perm = IOMMU_RW, + .translated_addr = 0xdeadbeefULL, + }; + IOMMUTLBEntry e2 = { + .iova = addr, + .addr_mask = 0xfffULL, + .pasid = 2, + .perm = IOMMU_RW, + .translated_addr = 0xb001ULL, + }; + uint16_t i; + + ATC *atc = atc_new(4096, 48); + + atc_create_address_space_cache(atc, e1.pasid); + atc_create_address_space_cache(atc, e2.pasid); + atc_update(atc, &e1); + atc_update(atc, &e2); + + for (i = 0; i <= MAX(e1.pasid, e2.pasid) + 1; ++i) { + if (i == e1.pasid || i == e2.pasid) { + continue; + } + assert_lookup_equals(atc, NULL, i, addr); + } + assert_lookup_equals(atc, &e1, e1.pasid, addr); + assert_lookup_equals(atc, &e1, e1.pasid, addr); + atc_destroy(atc); +} + +static void test_large_address(void) +{ + IOMMUTLBEntry e1 = { + .iova = 0xaaaaaaaaa000ULL, + .addr_mask = 0xfffULL, + .pasid = 8, + .perm = IOMMU_RW, + .translated_addr = 0x5eeeeeedULL, + }; + IOMMUTLBEntry e2 = { + .iova = 0x1f00baaaaabf000ULL, + .addr_mask = 0xfffULL, + .pasid = e1.pasid, + .perm = IOMMU_RW, + .translated_addr = 0xdeadbeefULL, + }; + + ATC *atc = atc_new(4096, 57); + + atc_create_address_space_cache(atc, e1.pasid); + atc_update(atc, &e1); + atc_update(atc, &e2); + + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); + atc_destroy(atc); +} + +static void test_bigger_page(void) +{ + IOMMUTLBEntry e1 = { + .iova = 0xaabbccdde000ULL, + .addr_mask = 0x1fffULL, + .pasid = 1, + .perm = IOMMU_RW, + .translated_addr = 0x5eeeeeedULL, + }; + hwaddr i; + + ATC *atc = atc_new(8192, 43); + + atc_create_address_space_cache(atc, e1.pasid); + atc_update(atc, &e1); + + i = e1.iova & (~e1.addr_mask); + assert_lookup_equals(atc, NULL, e1.pasid, i - 1); + while (i <= e1.iova + e1.addr_mask) { + assert_lookup_equals(atc, &e1, e1.pasid, i); + ++i; + } + assert_lookup_equals(atc, NULL, e1.pasid, i); + atc_destroy(atc); +} + +static void test_unknown_pasid(void) +{ + IOMMUTLBEntry e1 = { + .iova = 0xaabbccfff000ULL, + .addr_mask = 0xfffULL, + .pasid = 1, + .perm = IOMMU_RW, + .translated_addr = 0x5eeeeeedULL, + }; + + ATC *atc = atc_new(4096, 48); + g_assert(atc_update(atc, &e1) != 0); + assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); +} + +static void test_invalidation(void) +{ + static uint64_t page_size = 4096; + IOMMUTLBEntry e1 = { + .iova = 0xaabbccddf000ULL, + .addr_mask = 0xfffULL, + .pasid = 1, + .perm = IOMMU_RW, + .translated_addr = 0x5eeeeeedULL, + }; + IOMMUTLBEntry e2 = { + .iova = 0xffe00000ULL, + .addr_mask = 0x1fffffULL, + .pasid = 1, + .perm = IOMMU_RW, + .translated_addr = 0xb000001ULL, + }; + IOMMUTLBEntry e3; + + ATC *atc = atc_new(page_size , 48); + atc_create_address_space_cache(atc, e1.pasid); + + atc_update(atc, &e1); + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); + atc_invalidate(atc, &e1); + assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); + + atc_update(atc, &e1); + atc_update(atc, &e2); + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); + atc_invalidate(atc, &e2); + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); + assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); + + /* invalidate a huge page by invalidating a small region */ + for (hwaddr addr = e2.iova; addr <= (e2.iova + e2.addr_mask); + addr += page_size) { + atc_update(atc, &e2); + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); + e3 = (IOMMUTLBEntry){ + .iova = addr, + .addr_mask = page_size - 1, + .pasid = e2.pasid, + .perm = IOMMU_RW, + .translated_addr = 0, + }; + atc_invalidate(atc, &e3); + assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); + } +} + +static void test_delete_address_space_cache(void) +{ + static uint64_t page_size = 4096; + IOMMUTLBEntry e1 = { + .iova = 0xaabbccddf000ULL, + .addr_mask = 0xfffULL, + .pasid = 1, + .perm = IOMMU_RW, + .translated_addr = 0x5eeeeeedULL, + }; + IOMMUTLBEntry e2 = { + .iova = e1.iova, + .addr_mask = 0xfffULL, + .pasid = 2, + .perm = IOMMU_RW, + .translated_addr = 0x5eeeeeedULL, + }; + + ATC *atc = atc_new(page_size , 48); + atc_create_address_space_cache(atc, e1.pasid); + + atc_update(atc, &e1); + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); + atc_invalidate(atc, &e2); /* unkown pasid : is a nop*/ + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); + + atc_create_address_space_cache(atc, e2.pasid); + atc_update(atc, &e2); + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); + atc_invalidate(atc, &e1); + /* e1 has been removed but e2 is still there */ + assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); + + atc_update(atc, &e1); + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); + + atc_delete_address_space_cache(atc, e2.pasid); + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); + assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); +} + +static void test_invalidate_entire_address_space(void) +{ + static uint64_t page_size = 4096; + IOMMUTLBEntry e1 = { + .iova = 0x1000ULL, + .addr_mask = 0xfffULL, + .pasid = 1, + .perm = IOMMU_RW, + .translated_addr = 0x5eedULL, + }; + IOMMUTLBEntry e2 = { + .iova = 0xfffffffff000ULL, + .addr_mask = 0xfffULL, + .pasid = 1, + .perm = IOMMU_RW, + .translated_addr = 0xbeefULL, + }; + IOMMUTLBEntry e3 = { + .iova = 0, + .addr_mask = 0xffffffffffffffffULL, + .pasid = 1, + .perm = IOMMU_RW, + .translated_addr = 0, + }; + + ATC *atc = atc_new(page_size , 48); + atc_create_address_space_cache(atc, e1.pasid); + + atc_update(atc, &e1); + atc_update(atc, &e2); + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); + atc_invalidate(atc, &e3); + /* e1 has been removed but e2 is still there */ + assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); + assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); + + atc_destroy(atc); +} + +static void test_reset(void) +{ + static uint64_t page_size = 4096; + IOMMUTLBEntry e1 = { + .iova = 0x1000ULL, + .addr_mask = 0xfffULL, + .pasid = 1, + .perm = IOMMU_RW, + .translated_addr = 0x5eedULL, + }; + IOMMUTLBEntry e2 = { + .iova = 0xfffffffff000ULL, + .addr_mask = 0xfffULL, + .pasid = 2, + .perm = IOMMU_RW, + .translated_addr = 0xbeefULL, + }; + + ATC *atc = atc_new(page_size , 48); + atc_create_address_space_cache(atc, e1.pasid); + atc_create_address_space_cache(atc, e2.pasid); + atc_update(atc, &e1); + atc_update(atc, &e2); + + assert_lookup_equals(atc, &e1, e1.pasid, e1.iova); + assert_lookup_equals(atc, &e2, e2.pasid, e2.iova); + + atc_reset(atc); + + assert_lookup_equals(atc, NULL, e1.pasid, e1.iova); + assert_lookup_equals(atc, NULL, e2.pasid, e2.iova); +} + +static void test_get_max_number_of_pages(void) +{ + static uint64_t page_size = 4096; + hwaddr base = 0xc0fee000; /* aligned */ + ATC *atc = atc_new(page_size , 48); + g_assert(atc_get_max_number_of_pages(atc, base, page_size / 2) == 1); + g_assert(atc_get_max_number_of_pages(atc, base, page_size) == 1); + g_assert(atc_get_max_number_of_pages(atc, base, page_size + 1) == 2); + + g_assert(atc_get_max_number_of_pages(atc, base + 10, 1) == 1); + g_assert(atc_get_max_number_of_pages(atc, base + 10, page_size - 10) == 1); + g_assert(atc_get_max_number_of_pages(atc, base + 10, + page_size - 10 + 1) == 2); + g_assert(atc_get_max_number_of_pages(atc, base + 10, + page_size - 10 + 2) == 2); + + g_assert(atc_get_max_number_of_pages(atc, base + page_size - 1, 1) == 1); + g_assert(atc_get_max_number_of_pages(atc, base + page_size - 1, 2) == 2); + g_assert(atc_get_max_number_of_pages(atc, base + page_size - 1, 3) == 2); + + g_assert(atc_get_max_number_of_pages(atc, base + 10, page_size * 20) == 21); + g_assert(atc_get_max_number_of_pages(atc, base + 10, + (page_size * 20) + (page_size - 10)) + == 21); + g_assert(atc_get_max_number_of_pages(atc, base + 10, + (page_size * 20) + + (page_size - 10 + 1)) == 22); +} + +int main(int argc, char **argv) +{ + g_test_init(&argc, &argv, NULL); + g_test_add_func("/atc/test_creation_parameters", test_creation_parameters); + g_test_add_func("/atc/test_single_entry", test_single_entry); + g_test_add_func("/atc/test_page_boundaries", test_page_boundaries); + g_test_add_func("/atc/test_huge_page", test_huge_page); + g_test_add_func("/atc/test_pasid", test_pasid); + g_test_add_func("/atc/test_large_address", test_large_address); + g_test_add_func("/atc/test_bigger_page", test_bigger_page); + g_test_add_func("/atc/test_unknown_pasid", test_unknown_pasid); + g_test_add_func("/atc/test_invalidation", test_invalidation); + g_test_add_func("/atc/test_delete_address_space_cache", + test_delete_address_space_cache); + g_test_add_func("/atc/test_invalidate_entire_address_space", + test_invalidate_entire_address_space); + g_test_add_func("/atc/test_reset", test_reset); + g_test_add_func("/atc/test_get_max_number_of_pages", + test_get_max_number_of_pages); + return g_test_run(); +} diff --git a/util/atc.c b/util/atc.c new file mode 100644 index 0000000000..d951532e26 --- /dev/null +++ b/util/atc.c @@ -0,0 +1,211 @@ +/* + * QEMU emulation of an ATC + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License along + * with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#include "util/atc.h" + + +#define PAGE_TABLE_ENTRY_SIZE 8 + +/* a pasid is hashed using the identity function */ +static guint atc_pasid_key_hash(gconstpointer v) +{ + return (guint)(uintptr_t)v; /* pasid */ +} + +/* pasid equality */ +static gboolean atc_pasid_key_equal(gconstpointer v1, gconstpointer v2) +{ + return v1 == v2; +} + +/* Hash function for IOTLB entries */ +static guint atc_addr_key_hash(gconstpointer v) +{ + hwaddr addr = (hwaddr)v; + return (guint)((addr >> 32) ^ (addr & 0xffffffffU)); +} + +/* Equality test for IOTLB entries */ +static gboolean atc_addr_key_equal(gconstpointer v1, gconstpointer v2) +{ + return (hwaddr)v1 == (hwaddr)v2; +} + +static void atc_address_space_free(void *as) +{ + g_hash_table_unref(as); +} + +/* return log2(val), or UINT8_MAX if val is not a power of 2 */ +static uint8_t ilog2(uint64_t val) +{ + uint8_t result = 0; + while (val != 1) { + if (val & 1) { + return UINT8_MAX; + } + + val >>= 1; + result += 1; + } + return result; +} + +ATC *atc_new(uint64_t page_size, uint8_t address_width) +{ + ATC *atc; + uint8_t log_page_size = ilog2(page_size); + /* number of bits each used to store all the intermediate indexes */ + uint64_t addr_lookup_indexes_size; + + if (log_page_size == UINT8_MAX) { + return NULL; + } + /* + * We only support page table entries of 8 (PAGE_TABLE_ENTRY_SIZE) bytes + * log2(page_size / 8) = log2(page_size) - 3 + * is the level offset + */ + if (log_page_size <= 3) { + return NULL; + } + + atc = g_new0(ATC, 1); + atc->address_spaces = g_hash_table_new_full(atc_pasid_key_hash, + atc_pasid_key_equal, + NULL, atc_address_space_free); + atc->level_offset = log_page_size - 3; + /* at this point, we know that page_size is a power of 2 */ + atc->min_addr_mask = page_size - 1; + addr_lookup_indexes_size = address_width - log_page_size; + if ((addr_lookup_indexes_size % atc->level_offset) != 0) { + goto error; + } + atc->levels = addr_lookup_indexes_size / atc->level_offset; + atc->page_size = page_size; + return atc; + +error: + g_free(atc); + return NULL; +} + +static inline GHashTable *atc_get_address_space_cache(ATC *atc, uint32_t pasid) +{ + return g_hash_table_lookup(atc->address_spaces, + (gconstpointer)(uintptr_t)pasid); +} + +void atc_create_address_space_cache(ATC *atc, uint32_t pasid) +{ + GHashTable *as_cache; + + as_cache = atc_get_address_space_cache(atc, pasid); + if (!as_cache) { + as_cache = g_hash_table_new_full(atc_addr_key_hash, + atc_addr_key_equal, + NULL, g_free); + g_hash_table_replace(atc->address_spaces, + (gpointer)(uintptr_t)pasid, as_cache); + } +} + +void atc_delete_address_space_cache(ATC *atc, uint32_t pasid) +{ + g_hash_table_remove(atc->address_spaces, (gpointer)(uintptr_t)pasid); +} + +int atc_update(ATC *atc, IOMMUTLBEntry *entry) +{ + IOMMUTLBEntry *value; + GHashTable *as_cache = atc_get_address_space_cache(atc, entry->pasid); + if (!as_cache) { + return -ENODEV; + } + value = g_memdup2(entry, sizeof(*value)); + g_hash_table_replace(as_cache, (gpointer)(entry->iova), value); + return 0; +} + +IOMMUTLBEntry *atc_lookup(ATC *atc, uint32_t pasid, hwaddr addr) +{ + IOMMUTLBEntry *entry; + hwaddr mask = atc->min_addr_mask; + hwaddr key = addr & (~mask); + GHashTable *as_cache = atc_get_address_space_cache(atc, pasid); + + if (!as_cache) { + return NULL; + } + + /* + * Iterate over the possible page sizes and try to find a hit + */ + for (uint8_t level = 0; level < atc->levels; ++level) { + entry = g_hash_table_lookup(as_cache, (gconstpointer)key); + if (entry) { + return entry; + } + mask = (mask << atc->level_offset) | ((1 << atc->level_offset) - 1); + key = addr & (~mask); + } + + return NULL; +} + +static gboolean atc_invalidate_entry_predicate(gpointer key, gpointer value, + gpointer user_data) +{ + IOMMUTLBEntry *entry = (IOMMUTLBEntry *)value; + IOMMUTLBEntry *target = (IOMMUTLBEntry *)user_data; + hwaddr target_mask = ~target->addr_mask; + hwaddr entry_mask = ~entry->addr_mask; + return ((target->iova & target_mask) == (entry->iova & target_mask)) || + ((target->iova & entry_mask) == (entry->iova & entry_mask)); +} + +void atc_invalidate(ATC *atc, IOMMUTLBEntry *entry) +{ + GHashTable *as_cache = atc_get_address_space_cache(atc, entry->pasid); + if (!as_cache) { + return; + } + g_hash_table_foreach_remove(as_cache, + atc_invalidate_entry_predicate, + entry); +} + +void atc_destroy(ATC *atc) +{ + g_hash_table_unref(atc->address_spaces); +} + +size_t atc_get_max_number_of_pages(ATC *atc, hwaddr addr, size_t length) +{ + hwaddr page_mask = ~(atc->min_addr_mask); + size_t result = (length / atc->page_size); + if ((((addr & page_mask) + length - 1) & page_mask) != + ((addr + length - 1) & page_mask)) { + result += 1; + } + return result + (length % atc->page_size != 0 ? 1 : 0); +} + +void atc_reset(ATC *atc) +{ + g_hash_table_remove_all(atc->address_spaces); +} diff --git a/util/atc.h b/util/atc.h new file mode 100644 index 0000000000..8be95f5cca --- /dev/null +++ b/util/atc.h @@ -0,0 +1,117 @@ +/* + * QEMU emulation of an ATC + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License along + * with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef UTIL_ATC_H +#define UTIL_ATC_H + +#include "qemu/osdep.h" +#include "exec/memory.h" + +typedef struct ATC { + GHashTable *address_spaces; /* Key : pasid, value : GHashTable */ + hwaddr min_addr_mask; + uint64_t page_size; + uint8_t levels; + uint8_t level_offset; +} ATC; + +/* + * atc_new: Create an ATC. + * + * Return an ATC or NULL if the creation failed + * + * @page_size: #PCIDevice doing the memory access + * @address_width: width of the virtual addresses used by the IOMMU (in bits) + */ +ATC *atc_new(uint64_t page_size, uint8_t address_width); + +/* + * atc_update: Insert or update an entry in the cache + * + * Return 0 if the operation succeeds, a negative error code otherwise + * + * The insertion will fail if the address space associated with this pasid + * has not been created with atc_create_address_space_cache + * + * @atc: the ATC to update + * @entry: the tlb entry to insert into the cache + */ +int atc_update(ATC *atc, IOMMUTLBEntry *entry); + +/* + * atc_create_address_space_cache: delare a new address space + * identified by a PASID + * + * @atc: the ATC to update + * @pasid: the pasid of the address space to be created + */ +void atc_create_address_space_cache(ATC *atc, uint32_t pasid); + +/* + * atc_delete_address_space_cache: delete an address space + * identified by a PASID + * + * @atc: the ATC to update + * @pasid: the pasid of the address space to be deleted + */ +void atc_delete_address_space_cache(ATC *atc, uint32_t pasid); + +/* + * atc_lookup: query the cache in a given address space + * + * @atc: the ATC to query + * @pasid: the pasid of the address space to query + * @addr: the virtual address to translate + */ +IOMMUTLBEntry *atc_lookup(ATC *atc, uint32_t pasid, hwaddr addr); + +/* + * atc_invalidate: invalidate an entry in the cache + * + * @atc: the ATC to update + * @entry: the entry to invalidate + */ +void atc_invalidate(ATC *atc, IOMMUTLBEntry *entry); + +/* + * atc_destroy: delete an ATC + * + * @atc: the cache to be deleted + */ +void atc_destroy(ATC *atc); + +/* + * atc_get_max_number_of_pages: get the number of pages a memory operation + * will access if all the pages concerned have the minimum size. + * + * This function can be used to determine the size of the result array to be + * allocated when issuing an ATS request. + * + * @atc: the cache + * @addr: start address + * @length: number of bytes accessed from addr + */ +size_t atc_get_max_number_of_pages(ATC *atc, hwaddr addr, size_t length); + +/* + * atc_reset: invalidates all the entries stored in the ATC + * + * @atc: the cache + */ +void atc_reset(ATC *atc); + +#endif diff --git a/util/meson.build b/util/meson.build index 0ef9886be0..a2e0e9e5d7 100644 --- a/util/meson.build +++ b/util/meson.build @@ -94,6 +94,7 @@ if have_block util_ss.add(files('hbitmap.c')) util_ss.add(files('hexdump.c')) util_ss.add(files('iova-tree.c')) + util_ss.add(files('atc.c')) util_ss.add(files('iov.c', 'uri.c')) util_ss.add(files('nvdimm-utils.c')) util_ss.add(files('block-helpers.c'))
As the SVM-capable devices will need to cache translations, we provide an first implementation. This cache uses a two-level design based on hash tables. The first level is indexed by a PASID and the second by a virtual addresse. Signed-off-by: Clément Mathieu--Drif <clement.mathieu--drif@eviden.com> --- tests/unit/meson.build | 1 + tests/unit/test-atc.c | 502 +++++++++++++++++++++++++++++++++++++++++ util/atc.c | 211 +++++++++++++++++ util/atc.h | 117 ++++++++++ util/meson.build | 1 + 5 files changed, 832 insertions(+) create mode 100644 tests/unit/test-atc.c create mode 100644 util/atc.c create mode 100644 util/atc.h