diff --git a/.gitignore b/.gitignore
index ba74660..c833b86 100644
--- a/.gitignore
+++ b/.gitignore
@@ -42,6 +42,7 @@ htmlcov/
 nosetests.xml
 coverage.xml
 *,cover
+/.benchmarks/
 
 # Translations
 *.mo
diff --git a/README.rst b/README.rst
index 9c5263d..161869f 100644
--- a/README.rst
+++ b/README.rst
@@ -66,8 +66,26 @@ To run these tests yourself, clone the repo and run:
 
 .. code-block::
 
-   $ tox -e py34 -- -m benchmark
+   $ tox -e py34 -- -m benchmark --compare-other-engines
 
+Without the option ``--compare-other-engines`` it will focus only on ``RapidJSON``.
+This is particularly handy coupled with the `compare past runs`__ functionality of
+``pytest-benchmark``:
+
+.. code-block::
+
+   $ tox -e py34 -- -m benchmark --benchmark-autosave
+   # hack, hack, hack!
+   $ tox -e py34 -- -m benchmark --benchmark-compare=0001
+
+   ----------------------- benchmark 'deserialize': 18 tests ------------------------
+   Name (time in us)                                                            Min…
+   ----------------------------------------------------------------------------------
+   test_loads[rapidjson-256 Trues array] (NOW)                         5.2320 (1.0)…
+   test_loads[rapidjson-256 Trues array] (0001)                        5.4180 (1.04)…
+   …
+
+__ http://pytest-benchmark.readthedocs.org/en/latest/comparing.html
 
 Incompatibility
 ---------------
diff --git a/requirements-test.txt b/requirements-test.txt
index f9b2918..bca568b 100644
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -1,4 +1,5 @@
 pytest
+pytest-benchmark
 pytz
 simplejson
 ujson
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..3f1f341
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,107 @@
+from collections import namedtuple
+from functools import partial
+from operator import attrgetter
+
+Contender = namedtuple('Contender', 'name,dumps,loads')
+
+
+# See https://github.com/ionelmc/pytest-benchmark/issues/48
+
+def pytest_benchmark_group_stats(config, benchmarks, group_by):
+    result = {}
+    for bench in benchmarks:
+        if config.option.compare_other_engines:
+            engine, data_kind = bench.param.split('-')
+            if engine.endswith('not precise'):
+                group = result.setdefault("not precise floats: %s" % bench.group, [])
+            else:
+                group = result.setdefault("%s: %s" % (data_kind, bench.group), [])
+        else:
+            group = result.setdefault(bench.group, [])
+        group.append(bench)
+    return sorted(result.items())
+
+
+def pytest_addoption(parser):
+    parser.addoption('--compare-other-engines', action='store_true',
+                     help='compare against other JSON engines')
+
+
+contenders = []
+inaccurate_floats_contenders = []
+
+import rapidjson
+
+contenders.append(Contender('rapidjson',
+                            rapidjson.dumps,
+                            partial(rapidjson.loads, precise_float=True)))
+
+inaccurate_floats_contenders.append(Contender('rapidjson not precise',
+                                              rapidjson.dumps,
+                                              partial(rapidjson.loads,
+                                                      precise_float=False)))
+
+try:
+    import yajl
+except ImportError:
+    pass
+else:
+    contenders.append(Contender('yajl',
+                                yajl.Encoder().encode,
+                                yajl.Decoder().decode))
+
+try:
+    import simplejson
+except ImportError:
+    pass
+else:
+    contenders.append(Contender('simplejson',
+                                simplejson.dumps,
+                                simplejson.loads))
+
+try:
+    import json
+except ImportError:
+    pass
+else:
+    contenders.append(Contender('stdlib json',
+                                json.dumps,
+                                json.loads))
+
+try:
+    import ujson
+except ImportError:
+    pass
+else:
+    contenders.append(Contender('ujson',
+                                ujson.dumps,
+                                partial(ujson.loads, precise_float=True)))
+    inaccurate_floats_contenders.append(Contender('ujson not precise',
+                                                  ujson.dumps,
+                                                  partial(ujson.loads,
+                                                          precise_float=False)))
+
+
+def pytest_generate_tests(metafunc):
+    if 'contender' in metafunc.fixturenames:
+        if metafunc.config.option.compare_other_engines:
+            metafunc.parametrize('contender', contenders, ids=attrgetter('name'))
+        else:
+            metafunc.parametrize('contender', contenders[:1], ids=attrgetter('name'))
+
+    if 'inaccurate_floats_contender' in metafunc.fixturenames:
+        if metafunc.config.option.compare_other_engines:
+            metafunc.parametrize('inaccurate_floats_contender',
+                                 inaccurate_floats_contenders,
+                                 ids=attrgetter('name'))
+        else:
+            metafunc.parametrize('inaccurate_floats_contender',
+                                 inaccurate_floats_contenders[:1],
+                                 ids=attrgetter('name'))
+
+    if 'datetimes_loads_contender' in metafunc.fixturenames:
+        metafunc.parametrize('datetimes_loads_contender',
+                             [rapidjson.loads,
+                              partial(rapidjson.loads,
+                                      datetime_mode=rapidjson.DATETIME_MODE_ISO8601)],
+                             ids=['Ignore datetimes', 'Parse datetimes'])
diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py
index 47ba9a8..99b2731 100644
--- a/tests/test_benchmark.py
+++ b/tests/test_benchmark.py
@@ -1,36 +1,10 @@
 import datetime
 import pytest
-import time
 import sys
 import random
-from functools import partial
 
-try:
-    import yajl
-except ImportError:
-    yajl = None
 
-try:
-    import simplejson
-except ImportError:
-    simplejson = None
-
-try:
-    import json
-except ImportError:
-    json = None
-
-try:
-    import rapidjson
-except ImportError:
-    rapidjson = None
-
-try:
-    import ujson
-except ImportError:
-    ujson = None
-
-default_data = {
+composite_object = {
     'words': """
         Lorem ipsum dolor sit amet, consectetur adipiscing
         elit. Mauris adipiscing adipiscing placerat.
@@ -57,78 +31,15 @@
 
 friends = [ user, user, user, user, user, user, user, user ]
 
-
-def time_func(func, data, iterations):
-    start = time.time()
-    while iterations:
-        iterations -= 1
-        func(data)
-    return time.time() - start
-
-
-def run_client_test(
-        name, serialize, deserialize, iterations=100*1000, data=default_data
-):
-    squashed_data = serialize(data)
-    serialize_profile = time_func(serialize, data, iterations)
-    deserialize_profile = time_func(deserialize, squashed_data, iterations)
-    return serialize_profile, deserialize_profile
-
-contenders = []
-
-if yajl:
-    contenders.append(('yajl', yajl.Encoder().encode, yajl.Decoder().decode))
-
-if simplejson:
-    contenders.append(('simplejson', simplejson.dumps, simplejson.loads))
-
-if json:
-    contenders.append(('stdlib json', json.dumps, json.loads))
-
-if rapidjson:
-    contenders.append(
-        ('rapidjson', rapidjson.dumps,
-         partial(rapidjson.loads, precise_float=True))
-    )
-
-if ujson:
-    contenders.append(
-        ('ujson', ujson.dumps, partial(ujson.loads, precise_float=True))
-    )
-
-unprecise_rapid = (
-    'rapidjson (not precise)',
-    rapidjson.dumps,
-    partial(rapidjson.loads, precise_float=False)
-)
-
-unprecise_ujson = (
-    'ujson (not precise)',
-    ujson.dumps,
-    partial(ujson.loads, precise_float=False)
-)
-
-datetime_none_rapid = (
-    'rapidjson (dtm=None)',
-    partial(rapidjson.dumps, datetime_mode=None),
-    partial(rapidjson.loads, datetime_mode=None)
-)
-
-datetime_iso8601_rapid = (
-    'rapidjson (dtm=ISO8601)',
-    partial(rapidjson.dumps, datetime_mode=rapidjson.DATETIME_MODE_ISO8601),
-    partial(rapidjson.loads, datetime_mode=rapidjson.DATETIME_MODE_ISO8601)
-)
-
-
 doubles = []
 unicode_strings = []
 strings = []
 booleans = []
+datetimes = []
 list_dicts = []
 dict_lists = {}
 
-medium_complex = [
+complex_object = [
     [user, friends],  [user, friends],  [user, friends],
     [user, friends],  [user, friends],  [user, friends]
 ]
@@ -138,6 +49,13 @@ def run_client_test(
     unicode_strings.append("نظام الحكم سلطاني وراثي في الذكور من ذرية السيد تركي بن سعيد بن سلطان ويشترط فيمن يختار لولاية الحكم من بينهم ان يكون مسلما رشيدا عاقلا ًوابنا شرعيا لابوين عمانيين ")
     strings.append("A pretty long string which is in a list")
     booleans.append(True)
+    datetimes.append([datetime.date(random.randint(1900, 2000),
+                                    random.randint(1, 12),
+                                    random.randint(1, 28)),
+                      datetime.time(random.randint(0,23),
+                                    random.randint(0, 59),
+                                    random.randint(0, 59)),
+                      datetime.datetime.now()])
 
 for y in range(100):
     arrays = []
@@ -148,218 +66,44 @@ def run_client_test(
         dict_lists[str(random.random() * 20)] = arrays
 
 
-@pytest.mark.benchmark
-@pytest.mark.parametrize('name,serialize,deserialize', contenders)
-def test_json_serialization(name, serialize, deserialize):
-    ser_data, des_data = run_client_test(name, serialize, deserialize)
-    msg = "\n%-11s serialize: %0.3f  deserialize: %0.3f  total: %0.3f" % (
-        name, ser_data, des_data, ser_data + des_data
-    )
-
-    print(msg)
-
-
-@pytest.mark.benchmark
-@pytest.mark.parametrize(
-    'name,serialize,deserialize',
-    contenders + [unprecise_ujson, unprecise_rapid]
-)
-def test_json_doubles(name, serialize, deserialize):
-    print("\nArray with 256 doubles:")
-    ser_data, des_data = run_client_test(
-        name, serialize, deserialize,
-        data=doubles,
-        iterations=50000,
-    )
-    msg = "%-11s serialize: %0.3f  deserialize: %0.3f  total: %0.3f" % (
-        name, ser_data, des_data, ser_data + des_data
-    )
-
-    print(msg)
-
-
-@pytest.mark.benchmark
-@pytest.mark.parametrize('name,serialize,deserialize', contenders)
-def test_json_unicode_strings(name, serialize, deserialize):
-    print("\nArray with 256 unicode strings:")
-    ser_data, des_data = run_client_test(
-        name, serialize, deserialize,
-        data=unicode_strings,
-        iterations=5000,
-    )
-    msg = "%-11s serialize: %0.3f  deserialize: %0.3f  total: %0.3f" % (
-        name, ser_data, des_data, ser_data + des_data
-    )
-
-    print(msg)
-
-
-@pytest.mark.benchmark
-@pytest.mark.parametrize('name,serialize,deserialize', contenders)
-def test_json_scii_strings(name, serialize, deserialize):
-    print("\nArray with 256 ascii strings:")
-    ser_data, des_data = run_client_test(
-        name, serialize, deserialize,
-        data=strings,
-    )
-    msg = "%-11s serialize: %0.3f  deserialize: %0.3f  total: %0.3f" % (
-        name, ser_data, des_data, ser_data + des_data
-    )
-
-    print(msg)
-
-
-@pytest.mark.benchmark
-@pytest.mark.parametrize('name,serialize,deserialize', contenders)
-def test_json_booleans(name, serialize, deserialize):
-    print("\nArray with 256 True's:")
-    ser_data, des_data = run_client_test(
-        name, serialize, deserialize,
-        data=booleans,
-    )
-    msg = "%-11s serialize: %0.3f  deserialize: %0.3f  total: %0.3f" % (
-        name, ser_data, des_data, ser_data + des_data
-    )
-
-    print(msg)
-
-
-@pytest.mark.benchmark
-@pytest.mark.parametrize('name,serialize,deserialize', contenders)
-def test_json_list_of_dictionaries(name, serialize, deserialize):
-    print("\nArray of 100 dictionaries:")
-    ser_data, des_data = run_client_test(
-        name, serialize, deserialize,
-        data=list_dicts,
-        iterations=5,
-    )
-    msg = "%-11s serialize: %0.3f  deserialize: %0.3f  total: %0.3f" % (
-        name, ser_data, des_data, ser_data + des_data
-    )
-
-    print(msg)
-
-
-@pytest.mark.benchmark
-@pytest.mark.parametrize('name,serialize,deserialize', contenders)
-def test_json_dictionary_of_lists(name, serialize, deserialize):
-    print("\nDictionary of 100 Arrays:")
-    ser_data, des_data = run_client_test(
-        name, serialize, deserialize,
-        data=dict_lists,
-        iterations=5,
-    )
-    msg = "%-11s serialize: %0.3f  deserialize: %0.3f  total: %0.3f" % (
-        name, ser_data, des_data, ser_data + des_data
-    )
-
-    print(msg)
-
-
-@pytest.mark.benchmark
-@pytest.mark.parametrize(
-    'name,serialize,deserialize',
-    contenders + [datetime_none_rapid, datetime_iso8601_rapid])
-def test_json_medium_complex_objects(name, serialize, deserialize):
-    print("\n256 Medium Complex objects:")
-    ser_data, des_data = run_client_test(
-        name, serialize, deserialize,
-        data=medium_complex,
-        iterations=50000,
-    )
-    msg = "%-11s serialize: %0.3f  deserialize: %0.3f  total: %0.3f" % (
-        name, ser_data, des_data, ser_data + des_data
-    )
-
-    print(msg)
-
-
-@pytest.mark.benchmark
-def test_double_performance_float_precision():
-    from functools import partial
-
-    print("\nArray with 256 doubles:")
-    name = 'rapidjson (precise)'
-    serialize = rapidjson.dumps
-    deserialize = rapidjson.loads
-
-    ser_data, des_data = run_client_test(
-        name, serialize, deserialize,
-        data=doubles,
-        iterations=50000,
-    )
-    msg = "%-11s serialize: %0.3f  deserialize: %0.3f  total: %0.3f" % (
-        name, ser_data, des_data, ser_data + des_data
-    )
-
-    print(msg)
-
-    name = 'rapidjson (not precise)'
-    serialize = rapidjson.dumps
-    deserialize = partial(rapidjson.loads, precise_float=False)
-
-    ser_data, des_data = run_client_test(
-        name, serialize, deserialize,
-        data=doubles,
-        iterations=50000,
-    )
-    msg = "%-11s serialize: %0.3f  deserialize: %0.3f  total: %0.3f" % (
-        name, ser_data, des_data, ser_data + des_data
-    )
-    print(msg)
-
+datasets = [('composite object', composite_object),
+            ('256 doubles array', doubles),
+            ('256 unicode array', unicode_strings),
+            ('256 ascii array', strings),
+            ('256 Trues array', booleans),
+            ('100 dicts array', list_dicts),
+            ('100 arrays dict', dict_lists),
+            ('complex object', complex_object),
+]
 
-birthdays_native = []
-birthdays_string = []
 
-for f in friends:
-    ndata = dict(f)
-    ndata['birthday'] = datetime.date(random.randint(1900, 2000),
-                                      random.randint(1, 12),
-                                      random.randint(1, 28))
-    ndata['birthtime'] = datetime.time(random.randint(0,23),
-                                       random.randint(0, 59),
-                                       random.randint(0, 59))
-    ndata['timestamp'] = datetime.datetime.now()
-    birthdays_native.append(ndata)
+@pytest.mark.benchmark(group='serialize')
+@pytest.mark.parametrize('data', [d[1] for d in datasets], ids=[d[0] for d in datasets])
+def test_dumps(contender, data, benchmark):
+    benchmark(contender.dumps, data)
 
-    sdata = dict(ndata)
-    sdata['birthday'] = ndata['birthday'].isoformat()
-    sdata['birthtime'] = ndata['birthtime'].isoformat()
-    sdata['timestamp'] = ndata['timestamp'].isoformat()
-    birthdays_string.append(sdata)
 
+@pytest.mark.benchmark(group='deserialize')
+@pytest.mark.parametrize('data', [d[1] for d in datasets], ids=[d[0] for d in datasets])
+def test_loads(contender, data, benchmark):
+    data = contender.dumps(data)
+    benchmark(contender.loads, data)
 
-@pytest.mark.benchmark
-def test_datetime_performance():
-    from functools import partial
 
-    print("\nDatetimes:")
-    name = 'rapidjson (string)'
-    serialize = rapidjson.dumps
-    deserialize = rapidjson.loads
+# Special case 1: precise vs unprecise
 
-    ser_data, des_data = run_client_test(
-        name, serialize, deserialize,
-        data=birthdays_string,
-        iterations=50000,
-    )
-    msg = "%-11s serialize: %0.3f  deserialize: %0.3f  total: %0.3f" % (
-        name, ser_data, des_data, ser_data + des_data
-    )
+@pytest.mark.benchmark(group='deserialize')
+@pytest.mark.parametrize('data', [doubles], ids=['256 doubles array'])
+def test_loads_float(inaccurate_floats_contender, data, benchmark):
+    data = inaccurate_floats_contender.dumps(doubles)
+    benchmark(inaccurate_floats_contender.loads, data)
 
-    print(msg)
 
-    name = 'rapidjson (native)'
-    serialize = partial(rapidjson.dumps, datetime_mode=rapidjson.DATETIME_MODE_ISO8601)
-    deserialize = partial(rapidjson.loads, datetime_mode=rapidjson.DATETIME_MODE_ISO8601)
+# Special case 2: load datetimes as plain strings vs datetime.xxx instances
 
-    ser_data, des_data = run_client_test(
-        name, serialize, deserialize,
-        data=birthdays_native,
-        iterations=50000,
-    )
-    msg = "%-11s serialize: %0.3f  deserialize: %0.3f  total: %0.3f" % (
-        name, ser_data, des_data, ser_data + des_data
-    )
-    print(msg)
+@pytest.mark.benchmark(group='deserialize')
+@pytest.mark.parametrize('data', [datetimes], ids=['256x3 datetimes'])
+def test_loads_datetimes(datetimes_loads_contender, data, benchmark):
+    from rapidjson import dumps, DATETIME_MODE_ISO8601
+    data = dumps(data, datetime_mode=DATETIME_MODE_ISO8601)
+    benchmark(datetimes_loads_contender, data)