Skip to content

Commit

Permalink
feat(python): add Job.getChildrenValues
Browse files Browse the repository at this point in the history
Adds getChildrenValues method to the Python package's Job class, used to query returning values of
child jobs
  • Loading branch information
Arvinje authored and roggervalf committed Oct 23, 2024
1 parent a615c6d commit 9f23bdb
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 2 deletions.
6 changes: 5 additions & 1 deletion python/bullmq/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
if TYPE_CHECKING:
from bullmq.queue import Queue
from bullmq.types import JobOptions
from bullmq.utils import get_parent_key
from bullmq.utils import get_parent_key, parse_json_string_values

import json
import time
Expand Down Expand Up @@ -210,6 +210,10 @@ async def saveStacktrace(self, pipe, err:str):
def moveToWaitingChildren(self, token, opts:dict):
return self.scripts.moveToWaitingChildren(self.id, token, opts)

async def getChildrenValues(self):
results = await self.queue.client.hgetall(f"{self.queue.prefix}:{self.queue.name}:{self.id}:processed")
return parse_json_string_values(results)

@staticmethod
def fromJSON(queue: Queue, rawData: dict, jobId: str | None = None):
"""
Expand Down
4 changes: 4 additions & 0 deletions python/bullmq/utils.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import semver
import traceback
import json


def isRedisVersionLowerThan(current_version, minimum_version):
Expand All @@ -18,3 +19,6 @@ def extract_result(job_task, emit_callback):
def get_parent_key(opts: dict):
if opts:
return f"{opts.get('queue')}:{opts.get('id')}"

def parse_json_string_values(input_dict: dict[str, str]) -> dict[str, dict]:
return {key: json.loads(value) for key, value in input_dict.items()}
73 changes: 72 additions & 1 deletion python/tests/job_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@

import unittest

from bullmq import Queue, Job
from asyncio import Future
from bullmq import Queue, Job, Worker, FlowProducer
from uuid import uuid4

queueName = f"__test_queue__{uuid4().hex}"
Expand Down Expand Up @@ -103,5 +104,75 @@ async def test_promote_delayed_job(self):

await queue.close()

async def test_get_children_values(self):
child_job_name = 'child-job'
children_data = [
{"bar": None},
{"baz": 12.93},
{"qux": "string value"}
]
parent_queue_name = f"__test_parent_queue__{uuid4().hex}"

processing_children = Future()

processed_children = 0
async def process1(job: Job, token: str):
nonlocal processed_children
processed_children+=1
if processed_children == len(children_data):
processing_children.set_result(None)
return children_data[job.data.get("idx")]

processing_parent = Future()

async def process2(job: Job, token: str):
children_values = await job.getChildrenValues()
processing_parent.set_result(children_values)
return 1

parent_worker = Worker(parent_queue_name, process2)
children_worker = Worker(queueName, process1)

flow = FlowProducer()
await flow.add(
{
"name": 'parent-job',
"queueName": parent_queue_name,
"data": {},
"children": [
{"name": child_job_name, "data": {"idx": 0, "foo": 'bar'}, "queueName": queueName},
{"name": child_job_name, "data": {"idx": 1, "foo": 'baz'}, "queueName": queueName},
{"name": child_job_name, "data": {"idx": 2, "foo": 'qux'}, "queueName": queueName}
]
}
)

await processing_children
await processing_parent

def on_parent_processed(future):
self.assertIn(children_data[0], future.result().values())
self.assertIn(children_data[1], future.result().values())
self.assertIn(children_data[2], future.result().values())

processing_parent.add_done_callback(on_parent_processed)

await parent_worker.close()
await children_worker.close()
await flow.close()

parent_queue = Queue(parent_queue_name)
await parent_queue.pause()
await parent_queue.obliterate()
await parent_queue.close()

async def test_get_children_values_on_simple_jobs(self):
queue = Queue(queueName)
job = await queue.add("test", {"foo": "bar"}, {"delay": 1500})
children_values = await job.getChildrenValues()
self.assertEqual(children_values, {})

await queue.close()

if __name__ == '__main__':
unittest.main()

0 comments on commit 9f23bdb

Please sign in to comment.