diff mbox

[KVM-AUTOTEST,8/8] kvm_runtest_2.py: use pickle instead of shelve when loading/saving env

Message ID 1244234794-7844-9-git-send-email-lmr@redhat.com (mailing list archive)
State New, archived
Headers show

Commit Message

Lucas Meneghel Rodrigues June 5, 2009, 8:46 p.m. UTC 
pickle allows more control over the load/save process. Specifically, it
enables us to dump the contents of an object to disk without having to
unpickle it.

shelve, which uses pickle, seems to pickle and unpickle every time sync()
is called. This is bad for classes that need to be unpickled only once
per test (such a class will be introduced in a future patch).

Signed-off-by: Michael Goldish <mgoldish@redhat.com>
---
 client/tests/kvm/kvm.py |   35 +++++++++++++++++++++++++----------
 1 files changed, 25 insertions(+), 10 deletions(-)

Comments

Lucas Meneghel Rodrigues June 8, 2009, 3:21 p.m. UTC | #1 
On Fri, 2009-06-05 at 17:46 -0300, Lucas Meneghel Rodrigues wrote:
> pickle allows more control over the load/save process. Specifically, it
> enables us to dump the contents of an object to disk without having to
> unpickle it.
> 
> shelve, which uses pickle, seems to pickle and unpickle every time sync()
> is called. This is bad for classes that need to be unpickled only once
> per test (such a class will be introduced in a future patch).

Applied.

> Signed-off-by: Michael Goldish <mgoldish@redhat.com>
> ---
>  client/tests/kvm/kvm.py |   35 +++++++++++++++++++++++++----------
>  1 files changed, 25 insertions(+), 10 deletions(-)
> 
> diff --git a/client/tests/kvm/kvm.py b/client/tests/kvm/kvm.py
> index 1b9013c..a658425 100644
> --- a/client/tests/kvm/kvm.py
> +++ b/client/tests/kvm/kvm.py
> @@ -1,4 +1,4 @@
> -import sys, os, time, shelve, random, resource, logging
> +import sys, os, time, shelve, random, resource, logging, cPickle
>  from autotest_lib.client.bin import test
>  from autotest_lib.client.common_lib import error
>  
> @@ -10,6 +10,22 @@ class test_routine:
>          self.routine = None
>  
> 
> +def dump_env(obj, filename):
> +    file = open(filename, "w")
> +    cPickle.dump(obj, file)
> +    file.close()
> +
> +
> +def load_env(filename, default=None):
> +    try:
> +        file = open(filename, "r")
> +    except:
> +        return default
> +    obj = cPickle.load(file)
> +    file.close()
> +    return obj
> +
> +
>  class kvm(test.test):
>      """
>      Suite of KVM virtualization functional tests.
> @@ -62,12 +78,12 @@ class kvm(test.test):
>          keys = params.keys()
>          keys.sort()
>          for key in keys:
> -            logging.debug("    %s = %s" % (key, params[key]))
> +            logging.debug("    %s = %s", key, params[key])
>              self.write_test_keyval({key: params[key]})
>  
>          # Open the environment file
>          env_filename = os.path.join(self.bindir, "env")
> -        env = shelve.open(env_filename, writeback=True)
> +        env = load_env(env_filename, {})
>          logging.debug("Contents of environment: %s" % str(env))
>  
>          try:
> @@ -90,21 +106,20 @@ class kvm(test.test):
>  
>                  # Preprocess
>                  kvm_preprocessing.preprocess(self, params, env)
> -                env.sync()
> +                dump_env(env, env_filename)
>                  # Run the test function
>                  routine_obj.routine(self, params, env)
> -                env.sync()
> +                dump_env(env, env_filename)
>  
>              except Exception, e:
> -                logging.error("Test failed: %s" % e)
> +                logging.error("Test failed: %s", e)
>                  logging.debug("Postprocessing on error...")
>                  kvm_preprocessing.postprocess_on_error(self, params, env)
> -                env.sync()
> +                dump_env(env, env_filename)
>                  raise
>  
>          finally:
>              # Postprocess
>              kvm_preprocessing.postprocess(self, params, env)
> -            logging.debug("Contents of environment: %s" % str(env))
> -            env.sync()
> -            env.close()
> +            logging.debug("Contents of environment: %s", str(env))
> +            dump_env(env, env_filename)
Martin Bligh June 8, 2009, 4:10 p.m. UTC | #2 
On Fri, Jun 5, 2009 at 1:46 PM, Lucas Meneghel Rodrigues<lmr@redhat.com> wrote:
> pickle allows more control over the load/save process. Specifically, it
> enables us to dump the contents of an object to disk without having to
> unpickle it.
>
> shelve, which uses pickle, seems to pickle and unpickle every time sync()
> is called. This is bad for classes that need to be unpickled only once
> per test (such a class will be introduced in a future patch).
>

> +def dump_env(obj, filename):
> +    file = open(filename, "w")
> +    cPickle.dump(obj, file)
> +    file.close()

This seems like a strange function name - it's really pickling any
object, nothing specific to do with the environment?

> +
> +def load_env(filename, default=None):
> +    try:
> +        file = open(filename, "r")
> +    except:
> +        return default
> +    obj = cPickle.load(file)
> +    file.close()
> +    return obj
> +
> +
>  class kvm(test.test):
>     """
>     Suite of KVM virtualization functional tests.
> @@ -62,12 +78,12 @@ class kvm(test.test):
>         keys = params.keys()
>         keys.sort()
>         for key in keys:
> -            logging.debug("    %s = %s" % (key, params[key]))
> +            logging.debug("    %s = %s", key, params[key])
>             self.write_test_keyval({key: params[key]})
>
>         # Open the environment file
>         env_filename = os.path.join(self.bindir, "env")
> -        env = shelve.open(env_filename, writeback=True)
> +        env = load_env(env_filename, {})
>         logging.debug("Contents of environment: %s" % str(env))
>
>         try:
> @@ -90,21 +106,20 @@ class kvm(test.test):
>
>                 # Preprocess
>                 kvm_preprocessing.preprocess(self, params, env)
> -                env.sync()
> +                dump_env(env, env_filename)
>                 # Run the test function
>                 routine_obj.routine(self, params, env)
> -                env.sync()
> +                dump_env(env, env_filename)
>
>             except Exception, e:
> -                logging.error("Test failed: %s" % e)
> +                logging.error("Test failed: %s", e)
>                 logging.debug("Postprocessing on error...")
>                 kvm_preprocessing.postprocess_on_error(self, params, env)
> -                env.sync()
> +                dump_env(env, env_filename)
>                 raise
>
>         finally:
>             # Postprocess
>             kvm_preprocessing.postprocess(self, params, env)
> -            logging.debug("Contents of environment: %s" % str(env))
> -            env.sync()
> -            env.close()
> +            logging.debug("Contents of environment: %s", str(env))
> +            dump_env(env, env_filename)
> --
> 1.6.2.2
>
> _______________________________________________
> Autotest mailing list
> Autotest@test.kernel.org
> http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
>
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Lucas Meneghel Rodrigues June 8, 2009, 4:54 p.m. UTC | #3 
On Mon, 2009-06-08 at 09:10 -0700, Martin Bligh wrote:
> On Fri, Jun 5, 2009 at 1:46 PM, Lucas Meneghel Rodrigues<lmr@redhat.com> wrote:
> > pickle allows more control over the load/save process. Specifically, it
> > enables us to dump the contents of an object to disk without having to
> > unpickle it.
> >
> > shelve, which uses pickle, seems to pickle and unpickle every time sync()
> > is called. This is bad for classes that need to be unpickled only once
> > per test (such a class will be introduced in a future patch).
> >
> 
> > +def dump_env(obj, filename):
> > +    file = open(filename, "w")
> > +    cPickle.dump(obj, file)
> > +    file.close()
> 
> This seems like a strange function name - it's really pickling any
> object, nothing specific to do with the environment?

Yes, I agree it's pickling any object, but in the end, we are using this
file to unpickle data from our environment file, hence Michael choose
that name.

> > +
> > +def load_env(filename, default=None):
> > +    try:
> > +        file = open(filename, "r")
> > +    except:
> > +        return default
> > +    obj = cPickle.load(file)
> > +    file.close()
> > +    return obj
> > +
> > +
> >  class kvm(test.test):
> >     """
> >     Suite of KVM virtualization functional tests.
> > @@ -62,12 +78,12 @@ class kvm(test.test):
> >         keys = params.keys()
> >         keys.sort()
> >         for key in keys:
> > -            logging.debug("    %s = %s" % (key, params[key]))
> > +            logging.debug("    %s = %s", key, params[key])
> >             self.write_test_keyval({key: params[key]})
> >
> >         # Open the environment file
> >         env_filename = os.path.join(self.bindir, "env")
> > -        env = shelve.open(env_filename, writeback=True)
> > +        env = load_env(env_filename, {})
> >         logging.debug("Contents of environment: %s" % str(env))
> >
> >         try:
> > @@ -90,21 +106,20 @@ class kvm(test.test):
> >
> >                 # Preprocess
> >                 kvm_preprocessing.preprocess(self, params, env)
> > -                env.sync()
> > +                dump_env(env, env_filename)
> >                 # Run the test function
> >                 routine_obj.routine(self, params, env)
> > -                env.sync()
> > +                dump_env(env, env_filename)
> >
> >             except Exception, e:
> > -                logging.error("Test failed: %s" % e)
> > +                logging.error("Test failed: %s", e)
> >                 logging.debug("Postprocessing on error...")
> >                 kvm_preprocessing.postprocess_on_error(self, params, env)
> > -                env.sync()
> > +                dump_env(env, env_filename)
> >                 raise
> >
> >         finally:
> >             # Postprocess
> >             kvm_preprocessing.postprocess(self, params, env)
> > -            logging.debug("Contents of environment: %s" % str(env))
> > -            env.sync()
> > -            env.close()
> > +            logging.debug("Contents of environment: %s", str(env))
> > +            dump_env(env, env_filename)
> > --
> > 1.6.2.2
> >
> > _______________________________________________
> > Autotest mailing list
> > Autotest@test.kernel.org
> > http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
> >
diff mbox

Patch

diff --git a/client/tests/kvm/kvm.py b/client/tests/kvm/kvm.py
index 1b9013c..a658425 100644
--- a/client/tests/kvm/kvm.py
+++ b/client/tests/kvm/kvm.py
@@ -1,4 +1,4 @@ 
-import sys, os, time, shelve, random, resource, logging
+import sys, os, time, shelve, random, resource, logging, cPickle
 from autotest_lib.client.bin import test
 from autotest_lib.client.common_lib import error
 
@@ -10,6 +10,22 @@  class test_routine:
         self.routine = None
 
 
+def dump_env(obj, filename):
+    file = open(filename, "w")
+    cPickle.dump(obj, file)
+    file.close()
+
+
+def load_env(filename, default=None):
+    try:
+        file = open(filename, "r")
+    except:
+        return default
+    obj = cPickle.load(file)
+    file.close()
+    return obj
+
+
 class kvm(test.test):
     """
     Suite of KVM virtualization functional tests.
@@ -62,12 +78,12 @@  class kvm(test.test):
         keys = params.keys()
         keys.sort()
         for key in keys:
-            logging.debug("    %s = %s" % (key, params[key]))
+            logging.debug("    %s = %s", key, params[key])
             self.write_test_keyval({key: params[key]})
 
         # Open the environment file
         env_filename = os.path.join(self.bindir, "env")
-        env = shelve.open(env_filename, writeback=True)
+        env = load_env(env_filename, {})
         logging.debug("Contents of environment: %s" % str(env))
 
         try:
@@ -90,21 +106,20 @@  class kvm(test.test):
 
                 # Preprocess
                 kvm_preprocessing.preprocess(self, params, env)
-                env.sync()
+                dump_env(env, env_filename)
                 # Run the test function
                 routine_obj.routine(self, params, env)
-                env.sync()
+                dump_env(env, env_filename)
 
             except Exception, e:
-                logging.error("Test failed: %s" % e)
+                logging.error("Test failed: %s", e)
                 logging.debug("Postprocessing on error...")
                 kvm_preprocessing.postprocess_on_error(self, params, env)
-                env.sync()
+                dump_env(env, env_filename)
                 raise
 
         finally:
             # Postprocess
             kvm_preprocessing.postprocess(self, params, env)
-            logging.debug("Contents of environment: %s" % str(env))
-            env.sync()
-            env.close()
+            logging.debug("Contents of environment: %s", str(env))
+            dump_env(env, env_filename)