Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extend async tree benchmarks to cover eager task execution #279

Merged
merged 2 commits into from
May 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions doc/benchmarks.rst
Original file line number Diff line number Diff line change
Expand Up @@ -66,12 +66,15 @@ Async workload benchmark, which calls ``asyncio.gather()`` on a tree (6 levels d

* ``async_tree``: no actual async work at any leaf node.
* ``async_tree_io``: all leaf nodes simulate async IO workload (async sleep 50ms).
* ``async_tree_memoization``: all leaf nodes simulate async IO workload with 90% of
* ``async_tree_memoization``: all leaf nodes simulate async IO workload with 90% of
the data memoized.
* ``async_tree_cpu_io_mixed``: half of the leaf nodes simulate CPU-bound workload
(``math.factorial(500)``) and the other half simulate the same workload as the
(``math.factorial(500)``) and the other half simulate the same workload as the
``async_tree_memoization`` variant.

These benchmarks also have an "eager" flavor that uses asyncio eager task factory,
if available.


chameleon
---------
Expand Down
4 changes: 4 additions & 0 deletions pyperformance/data-files/benchmarks/MANIFEST
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ async_tree <local>
async_tree_cpu_io_mixed <local:async_tree>
async_tree_io <local:async_tree>
async_tree_memoization <local:async_tree>
async_tree_eager <local:async_tree>
async_tree_eager_cpu_io_mixed <local:async_tree>
async_tree_eager_io <local:async_tree>
async_tree_eager_memoization <local:async_tree>
asyncio_tcp <local>
asyncio_tcp_ssl <local:asyncio_tcp>
concurrent_imap <local>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
[tool.pyperformance]
name = "async_tree_cpu_io_mixed"
extra_opts = ["cpu_io_mixed"]

kumaraditya303 marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[tool.pyperformance]
name = "async_tree_eager"
extra_opts = ["eager"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[tool.pyperformance]
name = "async_tree_eager_cpu_io_mixed"
extra_opts = ["eager_cpu_io_mixed"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[tool.pyperformance]
name = "async_tree_eager_io"
extra_opts = ["eager_io"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[tool.pyperformance]
name = "async_tree_eager_memoization"
extra_opts = ["eager_memoization"]
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
[tool.pyperformance]
name = "async_tree_io"
extra_opts = ["io"]

Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
[tool.pyperformance]
name = "async_tree_memoization"
extra_opts = ["memoization"]

44 changes: 37 additions & 7 deletions pyperformance/data-files/benchmarks/bm_async_tree/run_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,14 @@

1) "none": No actual async work in the async tree.
2) "io": All leaf nodes simulate async IO workload (async sleep 50ms).
3) "memoization": All leaf nodes simulate async IO workload with 90% of
3) "memoization": All leaf nodes simulate async IO workload with 90% of
the data memoized
4) "cpu_io_mixed": Half of the leaf nodes simulate CPU-bound workload and
the other half simulate the same workload as the
4) "cpu_io_mixed": Half of the leaf nodes simulate CPU-bound workload and
the other half simulate the same workload as the
"memoization" variant.

All variants also have an "eager" flavor that uses
the asyncio eager task factory (if available).
"""


Expand Down Expand Up @@ -57,16 +60,32 @@ async def run(self):
await self.recurse(NUM_RECURSE_LEVELS)


class EagerMixin:
async def run(self):
loop = asyncio.get_running_loop()
if hasattr(asyncio, 'eager_task_factory'):
loop.set_task_factory(asyncio.eager_task_factory)
return await super().run()


class NoneAsyncTree(AsyncTree):
async def workload_func(self):
return


class EagerAsyncTree(EagerMixin, NoneAsyncTree):
pass


class IOAsyncTree(AsyncTree):
async def workload_func(self):
await self.mock_io_call()


class EagerIOAsyncTree(EagerMixin, IOAsyncTree):
pass


class MemoizationAsyncTree(AsyncTree):
async def workload_func(self):
# deterministic random, seed set in AsyncTree.__init__()
Expand All @@ -82,6 +101,10 @@ async def workload_func(self):
return data


class EagerMemoizationAsyncTree(EagerMixin, MemoizationAsyncTree):
pass


class CpuIoMixedAsyncTree(MemoizationAsyncTree):
async def workload_func(self):
# deterministic random, seed set in AsyncTree.__init__()
Expand All @@ -92,6 +115,10 @@ async def workload_func(self):
return await MemoizationAsyncTree.workload_func(self)


class EagerCpuIoMixedAsyncTree(EagerMixin, CpuIoMixedAsyncTree):
pass


def add_metadata(runner):
runner.metadata["description"] = "Async tree workloads."
runner.metadata["async_tree_recurse_levels"] = NUM_RECURSE_LEVELS
Expand All @@ -115,20 +142,24 @@ def add_parser_args(parser):
Determines which benchmark to run. Options:
1) "none": No actual async work in the async tree.
2) "io": All leaf nodes simulate async IO workload (async sleep 50ms).
3) "memoization": All leaf nodes simulate async IO workload with 90% of
3) "memoization": All leaf nodes simulate async IO workload with 90% of
the data memoized
4) "cpu_io_mixed": Half of the leaf nodes simulate CPU-bound workload and
the other half simulate the same workload as the
4) "cpu_io_mixed": Half of the leaf nodes simulate CPU-bound workload and
the other half simulate the same workload as the
"memoization" variant.
""",
)


BENCHMARKS = {
"none": NoneAsyncTree,
"eager": EagerAsyncTree,
"io": IOAsyncTree,
"eager_io": EagerIOAsyncTree,
"memoization": MemoizationAsyncTree,
"eager_memoization": EagerMemoizationAsyncTree,
"cpu_io_mixed": CpuIoMixedAsyncTree,
"eager_cpu_io_mixed": EagerCpuIoMixedAsyncTree,
}


Expand All @@ -142,4 +173,3 @@ def add_parser_args(parser):
async_tree_class = BENCHMARKS[benchmark]
async_tree = async_tree_class()
runner.bench_async_func(f"async_tree_{benchmark}", async_tree.run)

kumaraditya303 marked this conversation as resolved.
Show resolved Hide resolved