From patchwork Sat May 8 17:01:11 2010 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: =?utf-8?b?THVrw6HFoSBEb2t0b3I=?= X-Patchwork-Id: 97931 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter.kernel.org (8.14.3/8.14.3) with ESMTP id o48H1whb017278 for ; Sat, 8 May 2010 17:01:59 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754397Ab0EHRBg (ORCPT ); Sat, 8 May 2010 13:01:36 -0400 Received: from mx1.redhat.com ([209.132.183.28]:47121 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1754179Ab0EHRBZ (ORCPT ); Sat, 8 May 2010 13:01:25 -0400 Received: from int-mx04.intmail.prod.int.phx2.redhat.com (int-mx04.intmail.prod.int.phx2.redhat.com [10.5.11.17]) by mx1.redhat.com (8.13.8/8.13.8) with ESMTP id o48H1OBC025674 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-SHA bits=256 verify=OK); Sat, 8 May 2010 13:01:24 -0400 Received: from localhost.localdomain (ovpn01.gateway.prod.ext.phx2.redhat.com [10.5.9.1]) by int-mx04.intmail.prod.int.phx2.redhat.com (8.13.8/8.13.8) with ESMTP id o48H1BYY006901; Sat, 8 May 2010 13:01:21 -0400 From: Lukas Doktor To: ldoktor@redhat.com Cc: jzupka@redhat.com, kvm@vger.kernel.org, autotest@test.kernel.org, jasowang@redhat.com, lmr@redhat.com Subject: [PATCH 4/4] guest_reserve, host_reserve and tmpfs overhead automatic calculation based on used memory Date: Sat, 8 May 2010 18:01:11 +0100 Message-Id: <1273338071-8595-5-git-send-email-ldoktor@redhat.com> In-Reply-To: <[KVM_AUTOTEST][PATCH] KSM_overcommit: dynamic reserve calculation (2)> References: <[KVM_AUTOTEST][PATCH] KSM_overcommit: dynamic reserve calculation (2)> X-Scanned-By: MIMEDefang 2.67 on 10.5.11.17 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.3 (demeter.kernel.org [140.211.167.41]); Sat, 08 May 2010 17:01:59 +0000 (UTC) diff --git a/client/tests/kvm/scripts/allocator.py b/client/tests/kvm/scripts/allocator.py index 1036893..227745a 100755 --- a/client/tests/kvm/scripts/allocator.py +++ b/client/tests/kvm/scripts/allocator.py @@ -8,10 +8,12 @@ Auxiliary script used to allocate memory on guests. """ -import os, array, sys, struct, random, copy, inspect, tempfile, datetime +import os, array, sys, struct, random, copy, inspect, tempfile, datetime, math PAGE_SIZE = 4096 # machine page size +TMPFS_OVERHEAD = 0.0022 # overhead on 1MB of write data + class MemFill(object): """ @@ -32,7 +34,8 @@ class MemFill(object): self.tmpdp = tempfile.mkdtemp() ret_code = os.system("mount -o size=%dM tmpfs %s -t tmpfs" % - ((mem + 25), self.tmpdp)) + ((mem+math.ceil(mem*TMPFS_OVERHEAD)), + self.tmpdp)) if ret_code != 0: if os.getuid() != 0: print ("FAIL: Unable to mount tmpfs " @@ -42,7 +45,7 @@ class MemFill(object): else: self.f = tempfile.TemporaryFile(prefix='mem', dir=self.tmpdp) self.allocate_by = 'L' - self.npages = (mem * 1024 * 1024) / PAGE_SIZE + self.npages = ((mem * 1024 * 1024) / PAGE_SIZE) self.random_key = random_key self.static_value = static_value print "PASS: Initialization" @@ -83,7 +86,7 @@ class MemFill(object): @return: return array of bytes size PAGE_SIZE. """ a = array.array("B") - for i in range(PAGE_SIZE / a.itemsize): + for i in range((PAGE_SIZE / a.itemsize)): try: a.append(value) except: diff --git a/client/tests/kvm/tests/ksm_overcommit.py b/client/tests/kvm/tests/ksm_overcommit.py index 24d6643..8dc1722 100644 --- a/client/tests/kvm/tests/ksm_overcommit.py +++ b/client/tests/kvm/tests/ksm_overcommit.py @@ -348,12 +348,29 @@ def run_ksm_overcommit(test, params, env): # Main test code logging.info("Starting phase 0: Initialization") + # host_reserve: mem reserve kept for the host system to run - host_reserve = int(params.get("ksm_host_reserve", 512)) + host_reserve = int(params.get("ksm_host_reserve", -1)) + if (host_reserve == -1): + # default host_reserve = MemAvailable + one_minimal_guest(128MB) + # later we add 64MB per additional guest + host_reserve = ((utils.memtotal() - utils.read_from_meminfo("MemFree")) + / 1024 + 128) + # using default reserve + _host_reserve = True + else: + _host_reserve = False + # guest_reserve: mem reserve kept to avoid guest OS to kill processes - guest_reserve = int(params.get("ksm_guest_reserve", 1024)) - logging.debug("Memory reserved for host to run: %d", host_reserve) - logging.debug("Memory reserved for guest to run: %d", guest_reserve) + guest_reserve = int(params.get("ksm_guest_reserve", -1)) + if (guest_reserve == -1): + # default guest_reserve = minimal_system_mem(256MB) + # later we add tmpfs overhead + guest_reserve = 256 + # using default reserve + _guest_reserve = True + else: + _guest_reserve = False max_vms = int(params.get("max_vms", 2)) overcommit = float(params.get("ksm_overcommit_ratio", 2.0)) @@ -365,6 +382,10 @@ def run_ksm_overcommit(test, params, env): if (params['ksm_mode'] == "serial"): max_alloc = vmsc + if _host_reserve: + # First round of additional guest reserves + host_reserve += vmsc * 64 + _host_reserve = vmsc host_mem = (int(utils.memtotal()) / 1024 - host_reserve) @@ -412,6 +433,10 @@ def run_ksm_overcommit(test, params, env): if mem - guest_reserve - 1 > 3100: vmsc = int(math.ceil((host_mem * overcommit) / (3100 + guest_reserve))) + if _host_reserve: + host_reserve += (vmsc - _host_reserve) * 64 + host_mem -= (vmsc - _host_reserve) * 64 + _host_reserve = vmsc mem = int(math.floor(host_mem * overcommit / vmsc)) if os.popen("uname -i").readline().startswith("i386"): @@ -420,8 +445,16 @@ def run_ksm_overcommit(test, params, env): if mem > 3100 - 64: vmsc = int(math.ceil((host_mem * overcommit) / (3100 - 64.0))) + if _host_reserve: + host_reserve += (vmsc - _host_reserve) * 64 + host_mem -= (vmsc - _host_reserve) * 64 + _host_reserve = vmsc mem = int(math.floor(host_mem * overcommit / vmsc)) + # 0.055 represents OS + TMPFS additional reserve per guest ram MB + if _guest_reserve: + guest_reserve += math.ceil(mem * 0.055) + swap = int(utils.read_from_meminfo("SwapTotal")) / 1024 logging.debug("Overcommit = %f", overcommit) diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample index bb3646c..06eaa74 100644 --- a/client/tests/kvm/tests_base.cfg.sample +++ b/client/tests/kvm/tests_base.cfg.sample @@ -305,9 +305,9 @@ variants: ksm_overcommit_ratio = 3 # Max paralel runs machine ksm_parallel_ratio = 4 - # Host memory reserve - ksm_host_reserve = 512 - ksm_guest_reserve = 1024 + # Host memory reserve (default - best fit for used mem) + # ksm_host_reserve = 512 + # ksm_guest_reserve = 1024 variants: - ksm_serial: ksm_mode = "serial"