| Message ID | 1465813009-21390-3-git-send-email-liang.z.li@intel.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
On Mon, Jun 13, 2016 at 06:16:44PM +0800, Liang Li wrote: > virtio-balloon can make use of the amount of free memory to determine > the amount of memory to be filled in the balloon, but the amount of > free memory will be effected by the page cache, which can be reclaimed. > Drop the cache before getting the amount of free memory will be very > helpful to relect the exact amount of memroy that can be reclaimed. Can't we just extend stats to report "reclaimable" memory? > This patch add a new feature to the balloon device to support this > operation, hypervisor can request the VM to drop it's cache, so as to > reclaim more memory. > > Signed-off-by: Liang Li <liang.z.li@intel.com> > --- > balloon.c | 10 ++- > hw/virtio/virtio-balloon.c | 85 ++++++++++++++++++++++++- > include/hw/virtio/virtio-balloon.h | 19 +++++- > include/standard-headers/linux/virtio_balloon.h | 1 + > include/sysemu/balloon.h | 5 +- > 5 files changed, 115 insertions(+), 5 deletions(-) > > diff --git a/balloon.c b/balloon.c > index f2ef50c..0fb34bf 100644 > --- a/balloon.c > +++ b/balloon.c > @@ -36,6 +36,7 @@ > > static QEMUBalloonEvent *balloon_event_fn; > static QEMUBalloonStatus *balloon_stat_fn; > +static QEMUBalloonDropCache *balloon_drop_cache_fn; > static void *balloon_opaque; > static bool balloon_inhibited; > > @@ -65,9 +66,12 @@ static bool have_balloon(Error **errp) > } > > int qemu_add_balloon_handler(QEMUBalloonEvent *event_func, > - QEMUBalloonStatus *stat_func, void *opaque) > + QEMUBalloonStatus *stat_func, > + QEMUBalloonDropCache *drop_cache_func, > + void *opaque) > { > - if (balloon_event_fn || balloon_stat_fn || balloon_opaque) { > + if (balloon_event_fn || balloon_stat_fn || balloon_drop_cache_fn > + || balloon_opaque) { > /* We're already registered one balloon handler. How many can > * a guest really have? > */ > @@ -75,6 +79,7 @@ int qemu_add_balloon_handler(QEMUBalloonEvent *event_func, > } > balloon_event_fn = event_func; > balloon_stat_fn = stat_func; > + balloon_drop_cache_fn = drop_cache_func; > balloon_opaque = opaque; > return 0; > } > @@ -86,6 +91,7 @@ void qemu_remove_balloon_handler(void *opaque) > } > balloon_event_fn = NULL; > balloon_stat_fn = NULL; > + balloon_drop_cache_fn = NULL; > balloon_opaque = NULL; > } > > diff --git a/hw/virtio/virtio-balloon.c b/hw/virtio/virtio-balloon.c > index 8cf74c2..4757ba5 100644 > --- a/hw/virtio/virtio-balloon.c > +++ b/hw/virtio/virtio-balloon.c > @@ -36,6 +36,10 @@ > > #define BALLOON_PAGE_SIZE (1 << VIRTIO_BALLOON_PFN_SHIFT) > > +enum balloon_req_id { > + BALLOON_DROP_CACHE, > +}; > + > static void balloon_page(void *addr, int deflate) > { > #if defined(__linux__) > @@ -154,6 +158,12 @@ static bool balloon_page_bitmap_supported(const VirtIOBalloon *s) > return virtio_vdev_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_BITMAP); > } > > +static bool balloon_misc_supported(const VirtIOBalloon *s) > +{ > + VirtIODevice *vdev = VIRTIO_DEVICE(s); > + return virtio_vdev_has_feature(vdev, VIRTIO_BALLOON_F_MISC); > +} > + > static bool balloon_stats_enabled(const VirtIOBalloon *s) > { > return s->stats_poll_interval > 0; > @@ -420,6 +430,39 @@ out: > } > } > > +static void virtio_balloon_handle_resp(VirtIODevice *vdev, VirtQueue *vq) > +{ > + VirtIOBalloon *s = VIRTIO_BALLOON(vdev); > + VirtQueueElement *elem; > + size_t offset = 0; > + uint32_t tmp32, id = 0; > + > + elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); > + if (!elem) { > + s->req_status = REQ_ERROR; > + return; > + } > + > + s->misc_vq_elem = elem; > + > + if (!elem->out_num) { > + return; > + } > + > + iov_to_buf(elem->out_sg, elem->out_num, offset, > + &tmp32, sizeof(uint32_t)); > + id = virtio_ldl_p(vdev, &tmp32); > + offset += sizeof(uint32_t); > + switch (id) { > + case BALLOON_DROP_CACHE: > + s->req_status = REQ_DONE; > + break; > + default: > + break; > + } > + > +} > + > static void virtio_balloon_get_config(VirtIODevice *vdev, uint8_t *config_data) > { > VirtIOBalloon *dev = VIRTIO_BALLOON(vdev); > @@ -490,6 +533,7 @@ static uint64_t virtio_balloon_get_features(VirtIODevice *vdev, uint64_t f, > f |= dev->host_features; > virtio_add_feature(&f, VIRTIO_BALLOON_F_STATS_VQ); > virtio_add_feature(&f, VIRTIO_BALLOON_F_PAGE_BITMAP); > + virtio_add_feature(&f, VIRTIO_BALLOON_F_MISC); > return f; > } > > @@ -500,6 +544,36 @@ static void virtio_balloon_stat(void *opaque, BalloonInfo *info) > VIRTIO_BALLOON_PFN_SHIFT); > } > > +static int virtio_balloon_drop_cache(void *opaque, unsigned long type) > +{ > + VirtIOBalloon *s = opaque; > + VirtIODevice *vdev = VIRTIO_DEVICE(s); > + VirtQueueElement *elem = s->misc_vq_elem; > + int len; > + > + if (!balloon_misc_supported(s)) { > + return REQ_UNSUPPORT; > + } > + > + if (elem == NULL || !elem->in_num) { > + elem = virtqueue_pop(s->mvq, sizeof(VirtQueueElement)); > + if (!elem) { > + return REQ_ERROR; > + } > + s->misc_vq_elem = elem; > + } > + s->misc_req.id = BALLOON_DROP_CACHE; > + s->misc_req.param = type; > + len = iov_from_buf(elem->in_sg, elem->in_num, 0, &s->misc_req, > + sizeof(s->misc_req)); > + virtqueue_push(s->mvq, elem, len); > + virtio_notify(vdev, s->mvq); > + g_free(s->misc_vq_elem); > + s->misc_vq_elem = NULL; > + > + return REQ_DONE; > +} > + > static void virtio_balloon_to_target(void *opaque, ram_addr_t target) > { > VirtIOBalloon *dev = VIRTIO_BALLOON(opaque); > @@ -562,7 +636,8 @@ static void virtio_balloon_device_realize(DeviceState *dev, Error **errp) > sizeof(struct virtio_balloon_config)); > > ret = qemu_add_balloon_handler(virtio_balloon_to_target, > - virtio_balloon_stat, s); > + virtio_balloon_stat, > + virtio_balloon_drop_cache, s); > > if (ret < 0) { > error_setg(errp, "Only one balloon device is supported"); > @@ -573,8 +648,10 @@ static void virtio_balloon_device_realize(DeviceState *dev, Error **errp) > s->ivq = virtio_add_queue(vdev, 128, virtio_balloon_handle_output); > s->dvq = virtio_add_queue(vdev, 128, virtio_balloon_handle_output); > s->svq = virtio_add_queue(vdev, 128, virtio_balloon_receive_stats); > + s->mvq = virtio_add_queue(vdev, 128, virtio_balloon_handle_resp); > > reset_stats(s); > + s->req_status = REQ_INIT; > > register_savevm(dev, "virtio-balloon", -1, 1, > virtio_balloon_save, virtio_balloon_load, s); > @@ -599,6 +676,12 @@ static void virtio_balloon_device_reset(VirtIODevice *vdev) > g_free(s->stats_vq_elem); > s->stats_vq_elem = NULL; > } > + > + if (s->misc_vq_elem != NULL) { > + g_free(s->misc_vq_elem); > + s->misc_vq_elem = NULL; > + } > + s->req_status = REQ_INIT; > } > > static void virtio_balloon_instance_init(Object *obj) > diff --git a/include/hw/virtio/virtio-balloon.h b/include/hw/virtio/virtio-balloon.h > index 35f62ac..a21bb45 100644 > --- a/include/hw/virtio/virtio-balloon.h > +++ b/include/hw/virtio/virtio-balloon.h > @@ -23,6 +23,20 @@ > #define VIRTIO_BALLOON(obj) \ > OBJECT_CHECK(VirtIOBalloon, (obj), TYPE_VIRTIO_BALLOON) > > +typedef enum { > + REQ_INIT, > + REQ_ON_GOING, > + REQ_DONE, > + REQ_ERROR, > + REQ_INVALID_PARAM, > + REQ_UNSUPPORT, > +} BalloonReqStatus; > + > +typedef struct GetFreePageReq { > + uint32_t id; > + uint32_t param; > +} MiscReq; > + > typedef struct virtio_balloon_stat VirtIOBalloonStat; > > typedef struct virtio_balloon_stat_modern { > @@ -33,16 +47,19 @@ typedef struct virtio_balloon_stat_modern { > > typedef struct VirtIOBalloon { > VirtIODevice parent_obj; > - VirtQueue *ivq, *dvq, *svq; > + VirtQueue *ivq, *dvq, *svq, *mvq; > uint32_t num_pages; > uint32_t actual; > uint64_t stats[VIRTIO_BALLOON_S_NR]; > VirtQueueElement *stats_vq_elem; > + VirtQueueElement *misc_vq_elem; > size_t stats_vq_offset; > QEMUTimer *stats_timer; > int64_t stats_last_update; > int64_t stats_poll_interval; > uint32_t host_features; > + MiscReq misc_req; > + BalloonReqStatus req_status; > } VirtIOBalloon; > > #endif > diff --git a/include/standard-headers/linux/virtio_balloon.h b/include/standard-headers/linux/virtio_balloon.h > index 7c9686c..c8b254f 100644 > --- a/include/standard-headers/linux/virtio_balloon.h > +++ b/include/standard-headers/linux/virtio_balloon.h > @@ -35,6 +35,7 @@ > #define VIRTIO_BALLOON_F_STATS_VQ 1 /* Memory Stats virtqueue */ > #define VIRTIO_BALLOON_F_DEFLATE_ON_OOM 2 /* Deflate balloon on OOM */ > #define VIRTIO_BALLOON_F_PAGE_BITMAP 3 /* Use page bitmap to send page info */ > +#define VIRTIO_BALLOON_F_MISC 4 /* Send request and get misc info */ > > /* Size of a PFN in the balloon interface. */ > #define VIRTIO_BALLOON_PFN_SHIFT 12 > diff --git a/include/sysemu/balloon.h b/include/sysemu/balloon.h > index 3f976b4..0e85f2b 100644 > --- a/include/sysemu/balloon.h > +++ b/include/sysemu/balloon.h > @@ -18,9 +18,12 @@ > > typedef void (QEMUBalloonEvent)(void *opaque, ram_addr_t target); > typedef void (QEMUBalloonStatus)(void *opaque, BalloonInfo *info); > +typedef int (QEMUBalloonDropCache)(void *opaque, unsigned long ctrl); > > int qemu_add_balloon_handler(QEMUBalloonEvent *event_func, > - QEMUBalloonStatus *stat_func, void *opaque); > + QEMUBalloonStatus *stat_func, > + QEMUBalloonDropCache *drop_cache_func, > + void *opaque); > void qemu_remove_balloon_handler(void *opaque); > bool qemu_balloon_is_inhibited(void); > void qemu_balloon_inhibit(bool state); > -- > 1.9.1
> On Mon, Jun 13, 2016 at 06:16:44PM +0800, Liang Li wrote: > > virtio-balloon can make use of the amount of free memory to determine > > the amount of memory to be filled in the balloon, but the amount of > > free memory will be effected by the page cache, which can be reclaimed. > > Drop the cache before getting the amount of free memory will be very > > helpful to relect the exact amount of memroy that can be reclaimed. > > Can't we just extend stats to report "reclaimable" memory? > Yes, I noticed the VIRTIO_BALLOON_S_AVAIL is for this purpose. I summarized the possible solutions from others: a. Drop the cache in guest agent instead of an obvious qmp command. (Paolo) b. Use a parameter as a hint to tell the guest live migration is going to happen, and let the guest do what it can do to make the host's life easier. (David) What' your opinion about these two solutions? Thanks! Liang
diff --git a/balloon.c b/balloon.c index f2ef50c..0fb34bf 100644 --- a/balloon.c +++ b/balloon.c @@ -36,6 +36,7 @@ static QEMUBalloonEvent *balloon_event_fn; static QEMUBalloonStatus *balloon_stat_fn; +static QEMUBalloonDropCache *balloon_drop_cache_fn; static void *balloon_opaque; static bool balloon_inhibited; @@ -65,9 +66,12 @@ static bool have_balloon(Error **errp) } int qemu_add_balloon_handler(QEMUBalloonEvent *event_func, - QEMUBalloonStatus *stat_func, void *opaque) + QEMUBalloonStatus *stat_func, + QEMUBalloonDropCache *drop_cache_func, + void *opaque) { - if (balloon_event_fn || balloon_stat_fn || balloon_opaque) { + if (balloon_event_fn || balloon_stat_fn || balloon_drop_cache_fn + || balloon_opaque) { /* We're already registered one balloon handler. How many can * a guest really have? */ @@ -75,6 +79,7 @@ int qemu_add_balloon_handler(QEMUBalloonEvent *event_func, } balloon_event_fn = event_func; balloon_stat_fn = stat_func; + balloon_drop_cache_fn = drop_cache_func; balloon_opaque = opaque; return 0; } @@ -86,6 +91,7 @@ void qemu_remove_balloon_handler(void *opaque) } balloon_event_fn = NULL; balloon_stat_fn = NULL; + balloon_drop_cache_fn = NULL; balloon_opaque = NULL; } diff --git a/hw/virtio/virtio-balloon.c b/hw/virtio/virtio-balloon.c index 8cf74c2..4757ba5 100644 --- a/hw/virtio/virtio-balloon.c +++ b/hw/virtio/virtio-balloon.c @@ -36,6 +36,10 @@ #define BALLOON_PAGE_SIZE (1 << VIRTIO_BALLOON_PFN_SHIFT) +enum balloon_req_id { + BALLOON_DROP_CACHE, +}; + static void balloon_page(void *addr, int deflate) { #if defined(__linux__) @@ -154,6 +158,12 @@ static bool balloon_page_bitmap_supported(const VirtIOBalloon *s) return virtio_vdev_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_BITMAP); } +static bool balloon_misc_supported(const VirtIOBalloon *s) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(s); + return virtio_vdev_has_feature(vdev, VIRTIO_BALLOON_F_MISC); +} + static bool balloon_stats_enabled(const VirtIOBalloon *s) { return s->stats_poll_interval > 0; @@ -420,6 +430,39 @@ out: } } +static void virtio_balloon_handle_resp(VirtIODevice *vdev, VirtQueue *vq) +{ + VirtIOBalloon *s = VIRTIO_BALLOON(vdev); + VirtQueueElement *elem; + size_t offset = 0; + uint32_t tmp32, id = 0; + + elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); + if (!elem) { + s->req_status = REQ_ERROR; + return; + } + + s->misc_vq_elem = elem; + + if (!elem->out_num) { + return; + } + + iov_to_buf(elem->out_sg, elem->out_num, offset, + &tmp32, sizeof(uint32_t)); + id = virtio_ldl_p(vdev, &tmp32); + offset += sizeof(uint32_t); + switch (id) { + case BALLOON_DROP_CACHE: + s->req_status = REQ_DONE; + break; + default: + break; + } + +} + static void virtio_balloon_get_config(VirtIODevice *vdev, uint8_t *config_data) { VirtIOBalloon *dev = VIRTIO_BALLOON(vdev); @@ -490,6 +533,7 @@ static uint64_t virtio_balloon_get_features(VirtIODevice *vdev, uint64_t f, f |= dev->host_features; virtio_add_feature(&f, VIRTIO_BALLOON_F_STATS_VQ); virtio_add_feature(&f, VIRTIO_BALLOON_F_PAGE_BITMAP); + virtio_add_feature(&f, VIRTIO_BALLOON_F_MISC); return f; } @@ -500,6 +544,36 @@ static void virtio_balloon_stat(void *opaque, BalloonInfo *info) VIRTIO_BALLOON_PFN_SHIFT); } +static int virtio_balloon_drop_cache(void *opaque, unsigned long type) +{ + VirtIOBalloon *s = opaque; + VirtIODevice *vdev = VIRTIO_DEVICE(s); + VirtQueueElement *elem = s->misc_vq_elem; + int len; + + if (!balloon_misc_supported(s)) { + return REQ_UNSUPPORT; + } + + if (elem == NULL || !elem->in_num) { + elem = virtqueue_pop(s->mvq, sizeof(VirtQueueElement)); + if (!elem) { + return REQ_ERROR; + } + s->misc_vq_elem = elem; + } + s->misc_req.id = BALLOON_DROP_CACHE; + s->misc_req.param = type; + len = iov_from_buf(elem->in_sg, elem->in_num, 0, &s->misc_req, + sizeof(s->misc_req)); + virtqueue_push(s->mvq, elem, len); + virtio_notify(vdev, s->mvq); + g_free(s->misc_vq_elem); + s->misc_vq_elem = NULL; + + return REQ_DONE; +} + static void virtio_balloon_to_target(void *opaque, ram_addr_t target) { VirtIOBalloon *dev = VIRTIO_BALLOON(opaque); @@ -562,7 +636,8 @@ static void virtio_balloon_device_realize(DeviceState *dev, Error **errp) sizeof(struct virtio_balloon_config)); ret = qemu_add_balloon_handler(virtio_balloon_to_target, - virtio_balloon_stat, s); + virtio_balloon_stat, + virtio_balloon_drop_cache, s); if (ret < 0) { error_setg(errp, "Only one balloon device is supported"); @@ -573,8 +648,10 @@ static void virtio_balloon_device_realize(DeviceState *dev, Error **errp) s->ivq = virtio_add_queue(vdev, 128, virtio_balloon_handle_output); s->dvq = virtio_add_queue(vdev, 128, virtio_balloon_handle_output); s->svq = virtio_add_queue(vdev, 128, virtio_balloon_receive_stats); + s->mvq = virtio_add_queue(vdev, 128, virtio_balloon_handle_resp); reset_stats(s); + s->req_status = REQ_INIT; register_savevm(dev, "virtio-balloon", -1, 1, virtio_balloon_save, virtio_balloon_load, s); @@ -599,6 +676,12 @@ static void virtio_balloon_device_reset(VirtIODevice *vdev) g_free(s->stats_vq_elem); s->stats_vq_elem = NULL; } + + if (s->misc_vq_elem != NULL) { + g_free(s->misc_vq_elem); + s->misc_vq_elem = NULL; + } + s->req_status = REQ_INIT; } static void virtio_balloon_instance_init(Object *obj) diff --git a/include/hw/virtio/virtio-balloon.h b/include/hw/virtio/virtio-balloon.h index 35f62ac..a21bb45 100644 --- a/include/hw/virtio/virtio-balloon.h +++ b/include/hw/virtio/virtio-balloon.h @@ -23,6 +23,20 @@ #define VIRTIO_BALLOON(obj) \ OBJECT_CHECK(VirtIOBalloon, (obj), TYPE_VIRTIO_BALLOON) +typedef enum { + REQ_INIT, + REQ_ON_GOING, + REQ_DONE, + REQ_ERROR, + REQ_INVALID_PARAM, + REQ_UNSUPPORT, +} BalloonReqStatus; + +typedef struct GetFreePageReq { + uint32_t id; + uint32_t param; +} MiscReq; + typedef struct virtio_balloon_stat VirtIOBalloonStat; typedef struct virtio_balloon_stat_modern { @@ -33,16 +47,19 @@ typedef struct virtio_balloon_stat_modern { typedef struct VirtIOBalloon { VirtIODevice parent_obj; - VirtQueue *ivq, *dvq, *svq; + VirtQueue *ivq, *dvq, *svq, *mvq; uint32_t num_pages; uint32_t actual; uint64_t stats[VIRTIO_BALLOON_S_NR]; VirtQueueElement *stats_vq_elem; + VirtQueueElement *misc_vq_elem; size_t stats_vq_offset; QEMUTimer *stats_timer; int64_t stats_last_update; int64_t stats_poll_interval; uint32_t host_features; + MiscReq misc_req; + BalloonReqStatus req_status; } VirtIOBalloon; #endif diff --git a/include/standard-headers/linux/virtio_balloon.h b/include/standard-headers/linux/virtio_balloon.h index 7c9686c..c8b254f 100644 --- a/include/standard-headers/linux/virtio_balloon.h +++ b/include/standard-headers/linux/virtio_balloon.h @@ -35,6 +35,7 @@ #define VIRTIO_BALLOON_F_STATS_VQ 1 /* Memory Stats virtqueue */ #define VIRTIO_BALLOON_F_DEFLATE_ON_OOM 2 /* Deflate balloon on OOM */ #define VIRTIO_BALLOON_F_PAGE_BITMAP 3 /* Use page bitmap to send page info */ +#define VIRTIO_BALLOON_F_MISC 4 /* Send request and get misc info */ /* Size of a PFN in the balloon interface. */ #define VIRTIO_BALLOON_PFN_SHIFT 12 diff --git a/include/sysemu/balloon.h b/include/sysemu/balloon.h index 3f976b4..0e85f2b 100644 --- a/include/sysemu/balloon.h +++ b/include/sysemu/balloon.h @@ -18,9 +18,12 @@ typedef void (QEMUBalloonEvent)(void *opaque, ram_addr_t target); typedef void (QEMUBalloonStatus)(void *opaque, BalloonInfo *info); +typedef int (QEMUBalloonDropCache)(void *opaque, unsigned long ctrl); int qemu_add_balloon_handler(QEMUBalloonEvent *event_func, - QEMUBalloonStatus *stat_func, void *opaque); + QEMUBalloonStatus *stat_func, + QEMUBalloonDropCache *drop_cache_func, + void *opaque); void qemu_remove_balloon_handler(void *opaque); bool qemu_balloon_is_inhibited(void); void qemu_balloon_inhibit(bool state);
virtio-balloon can make use of the amount of free memory to determine the amount of memory to be filled in the balloon, but the amount of free memory will be effected by the page cache, which can be reclaimed. Drop the cache before getting the amount of free memory will be very helpful to relect the exact amount of memroy that can be reclaimed. This patch add a new feature to the balloon device to support this operation, hypervisor can request the VM to drop it's cache, so as to reclaim more memory. Signed-off-by: Liang Li <liang.z.li@intel.com> --- balloon.c | 10 ++- hw/virtio/virtio-balloon.c | 85 ++++++++++++++++++++++++- include/hw/virtio/virtio-balloon.h | 19 +++++- include/standard-headers/linux/virtio_balloon.h | 1 + include/sysemu/balloon.h | 5 +- 5 files changed, 115 insertions(+), 5 deletions(-)