@@ -34,8 +34,26 @@ struct MemoryAttributeManager {
struct MemoryAttributeManagerClass {
ObjectClass parent_class;
+
+ int (*state_change)(MemoryAttributeManager *mgr, uint64_t offset, uint64_t size,
+ bool to_private);
};
+static inline int memory_attribute_manager_state_change(MemoryAttributeManager *mgr, uint64_t offset,
+ uint64_t size, bool to_private)
+{
+ MemoryAttributeManagerClass *klass;
+
+ if (mgr == NULL) {
+ return 0;
+ }
+
+ klass = MEMORY_ATTRIBUTE_MANAGER_GET_CLASS(mgr);
+
+ g_assert(klass->state_change);
+ return klass->state_change(mgr, offset, size, to_private);
+}
+
int memory_attribute_manager_realize(MemoryAttributeManager *mgr, MemoryRegion *mr);
void memory_attribute_manager_unrealize(MemoryAttributeManager *mgr);
@@ -234,6 +234,191 @@ static int memory_attribute_rdm_replay_discarded(const RamDiscardManager *rdm,
memory_attribute_rdm_replay_cb);
}
+static bool memory_attribute_is_valid_range(MemoryAttributeManager *mgr,
+ uint64_t offset, uint64_t size)
+{
+ MemoryRegion *mr = mgr->mr;
+
+ g_assert(mr);
+
+ uint64_t region_size = memory_region_size(mr);
+ int block_size = memory_attribute_manager_get_block_size(mgr);
+
+ if (!QEMU_IS_ALIGNED(offset, block_size)) {
+ return false;
+ }
+ if (offset + size < offset || !size) {
+ return false;
+ }
+ if (offset >= region_size || offset + size > region_size) {
+ return false;
+ }
+ return true;
+}
+
+static void memory_attribute_notify_discard(MemoryAttributeManager *mgr,
+ uint64_t offset, uint64_t size)
+{
+ RamDiscardListener *rdl;
+
+ QLIST_FOREACH(rdl, &mgr->rdl_list, next) {
+ MemoryRegionSection tmp = *rdl->section;
+
+ if (!memory_region_section_intersect_range(&tmp, offset, size)) {
+ continue;
+ }
+ rdl->notify_discard(rdl, &tmp);
+ }
+}
+
+static int memory_attribute_notify_populate(MemoryAttributeManager *mgr,
+ uint64_t offset, uint64_t size)
+{
+ RamDiscardListener *rdl, *rdl2;
+ int ret = 0;
+
+ QLIST_FOREACH(rdl, &mgr->rdl_list, next) {
+ MemoryRegionSection tmp = *rdl->section;
+
+ if (!memory_region_section_intersect_range(&tmp, offset, size)) {
+ continue;
+ }
+ ret = rdl->notify_populate(rdl, &tmp);
+ if (ret) {
+ break;
+ }
+ }
+
+ if (ret) {
+ /* Notify all already-notified listeners. */
+ QLIST_FOREACH(rdl2, &mgr->rdl_list, next) {
+ MemoryRegionSection tmp = *rdl2->section;
+
+ if (rdl2 == rdl) {
+ break;
+ }
+ if (!memory_region_section_intersect_range(&tmp, offset, size)) {
+ continue;
+ }
+ rdl2->notify_discard(rdl2, &tmp);
+ }
+ }
+ return ret;
+}
+
+static bool memory_attribute_is_range_populated(MemoryAttributeManager *mgr,
+ uint64_t offset, uint64_t size)
+{
+ const int block_size = memory_attribute_manager_get_block_size(mgr);
+ const unsigned long first_bit = offset / block_size;
+ const unsigned long last_bit = first_bit + (size / block_size) - 1;
+ unsigned long found_bit;
+
+ /* We fake a shorter bitmap to avoid searching too far. */
+ found_bit = find_next_zero_bit(mgr->shared_bitmap, last_bit + 1, first_bit);
+ return found_bit > last_bit;
+}
+
+static bool memory_attribute_is_range_discarded(MemoryAttributeManager *mgr,
+ uint64_t offset, uint64_t size)
+{
+ const int block_size = memory_attribute_manager_get_block_size(mgr);
+ const unsigned long first_bit = offset / block_size;
+ const unsigned long last_bit = first_bit + (size / block_size) - 1;
+ unsigned long found_bit;
+
+ /* We fake a shorter bitmap to avoid searching too far. */
+ found_bit = find_next_bit(mgr->shared_bitmap, last_bit + 1, first_bit);
+ return found_bit > last_bit;
+}
+
+static int memory_attribute_state_change(MemoryAttributeManager *mgr, uint64_t offset,
+ uint64_t size, bool to_private)
+{
+ const int block_size = memory_attribute_manager_get_block_size(mgr);
+ const unsigned long first_bit = offset / block_size;
+ const unsigned long nbits = size / block_size;
+ const uint64_t end = offset + size;
+ unsigned long bit;
+ uint64_t cur;
+ int ret = 0;
+
+ if (!memory_attribute_is_valid_range(mgr, offset, size)) {
+ error_report("%s, invalid range: offset 0x%lx, size 0x%lx",
+ __func__, offset, size);
+ return -1;
+ }
+
+ if (to_private) {
+ if (memory_attribute_is_range_discarded(mgr, offset, size)) {
+ /* Already private */
+ } else if (!memory_attribute_is_range_populated(mgr, offset, size)) {
+ /* Unexpected mixture: process individual blocks */
+ for (cur = offset; cur < end; cur += block_size) {
+ bit = cur / block_size;
+ if (!test_bit(bit, mgr->shared_bitmap)) {
+ continue;
+ }
+ clear_bit(bit, mgr->shared_bitmap);
+ memory_attribute_notify_discard(mgr, cur, block_size);
+ }
+ } else {
+ /* Completely shared */
+ bitmap_clear(mgr->shared_bitmap, first_bit, nbits);
+ memory_attribute_notify_discard(mgr, offset, size);
+ }
+ } else {
+ if (memory_attribute_is_range_populated(mgr, offset, size)) {
+ /* Already shared */
+ } else if (!memory_attribute_is_range_discarded(mgr, offset, size)) {
+ /* Unexpected mixture: process individual blocks */
+ unsigned long *modified_bitmap = bitmap_new(nbits);
+
+ for (cur = offset; cur < end; cur += block_size) {
+ bit = cur / block_size;
+ if (test_bit(bit, mgr->shared_bitmap)) {
+ continue;
+ }
+ set_bit(bit, mgr->shared_bitmap);
+ ret = memory_attribute_notify_populate(mgr, cur, block_size);
+ if (!ret) {
+ set_bit(bit - first_bit, modified_bitmap);
+ continue;
+ }
+ clear_bit(bit, mgr->shared_bitmap);
+ break;
+ }
+
+ if (ret) {
+ /*
+ * Very unexpected: something went wrong. Revert to the old
+ * state, marking only the blocks as private that we converted
+ * to shared.
+ */
+ for (cur = offset; cur < end; cur += block_size) {
+ bit = cur / block_size;
+ if (!test_bit(bit - first_bit, modified_bitmap)) {
+ continue;
+ }
+ assert(test_bit(bit, mgr->shared_bitmap));
+ clear_bit(bit, mgr->shared_bitmap);
+ memory_attribute_notify_discard(mgr, cur, block_size);
+ }
+ }
+ g_free(modified_bitmap);
+ } else {
+ /* Complete private */
+ bitmap_set(mgr->shared_bitmap, first_bit, nbits);
+ ret = memory_attribute_notify_populate(mgr, offset, size);
+ if (ret) {
+ bitmap_clear(mgr->shared_bitmap, first_bit, nbits);
+ }
+ }
+ }
+
+ return ret;
+}
+
int memory_attribute_manager_realize(MemoryAttributeManager *mgr, MemoryRegion *mr)
{
uint64_t shared_bitmap_size;
@@ -272,8 +457,11 @@ static void memory_attribute_manager_finalize(Object *obj)
static void memory_attribute_manager_class_init(ObjectClass *oc, void *data)
{
+ MemoryAttributeManagerClass *mamc = MEMORY_ATTRIBUTE_MANAGER_CLASS(oc);
RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_CLASS(oc);
+ mamc->state_change = memory_attribute_state_change;
+
rdmc->get_min_granularity = memory_attribute_rdm_get_min_granularity;
rdmc->register_listener = memory_attribute_rdm_register_listener;
rdmc->unregister_listener = memory_attribute_rdm_unregister_listener;
Introduce a new state_change() callback in MemoryAttributeManagerClass to efficiently notify all registered RamDiscardListeners, including VFIO listeners about the memory conversion events in guest_memfd. The existing VFIO listener can dynamically DMA map/unmap the shared pages based on conversion types: - For conversions from shared to private, the VFIO system ensures the discarding of shared mapping from the IOMMU. - For conversions from private to shared, it triggers the population of the shared mapping into the IOMMU. Additionally, there could be some special conversion requests: - When a conversion request is made for a page already in the desired state, the helper simply returns success. - For requests involving a range partially in the desired state, only the necessary segments are converted, ensuring the entire range complies with the request efficiently. In this case, fallback to a "1 block at a time" handling. - In scenarios where a conversion request is declined by other systems, such as a failure from VFIO during notify_populate(), the helper will roll back the request, maintaining consistency. Note that the bitmap status is updated before the notifier callbacks so that the listener can handle the memory based on the latest status. Opportunistically introduce a helper to trigger the state_change() callback of the class. Signed-off-by: Chenyi Qiang <chenyi.qiang@intel.com> --- Changes in v3: - Move the bitmap update before notifier callbacks. - Call the notifier callbacks directly in notify_discard/populate() with the expectation that the request memory range is in the desired attribute. - For the case that only partial range in the desire status, handle the range with block_size granularity for ease of rollback (https://lore.kernel.org/qemu-devel/812768d7-a02d-4b29-95f3-fb7a125cf54e@redhat.com/) Changes in v2: - Do the alignment changes due to the rename to MemoryAttributeManager - Move the state_change() helper definition in this patch. --- include/system/memory-attribute-manager.h | 18 +++ system/memory-attribute-manager.c | 188 ++++++++++++++++++++++ 2 files changed, 206 insertions(+)