mirror of https://github.com/rq/rq.git
Restructure some code.
No functional change, but leave the BLPOP'ing to the Queue, as the queues know how to pop themselves.
This commit is contained in:
parent
652ced2580
commit
f492a5ae2b
|
@ -0,0 +1,5 @@
|
||||||
|
class NoQueueError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class NoMoreWorkError(Exception):
|
||||||
|
pass
|
26
rq/queue.py
26
rq/queue.py
|
@ -1,7 +1,7 @@
|
||||||
import uuid
|
import uuid
|
||||||
from pickle import loads, dumps
|
from pickle import loads, dumps
|
||||||
from .proxy import conn
|
from .proxy import conn
|
||||||
|
from .exceptions import NoMoreWorkError
|
||||||
|
|
||||||
class DelayedResult(object):
|
class DelayedResult(object):
|
||||||
def __init__(self, key):
|
def __init__(self, key):
|
||||||
|
@ -18,7 +18,6 @@ class DelayedResult(object):
|
||||||
return self._rv
|
return self._rv
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def to_queue_key(queue_name):
|
def to_queue_key(queue_name):
|
||||||
return 'rq:%s' % (queue_name,)
|
return 'rq:%s' % (queue_name,)
|
||||||
|
|
||||||
|
@ -56,5 +55,28 @@ class Queue(object):
|
||||||
s = conn.lpop(self.key)
|
s = conn.lpop(self.key)
|
||||||
return loads(s)
|
return loads(s)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _dequeue_any(cls, queues):
|
||||||
|
# Redis' BLPOP command takes multiple queue arguments, but LPOP can
|
||||||
|
# only take a single queue. Therefore, we need to loop over all
|
||||||
|
# queues manually, in order, and return None if no more work is
|
||||||
|
# available
|
||||||
|
for queue in queues:
|
||||||
|
value = conn.lpop(queue)
|
||||||
|
if value is not None:
|
||||||
|
return (queue, value)
|
||||||
|
return None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def dequeue_any(cls, queues, blocking):
|
||||||
|
if blocking:
|
||||||
|
queue, msg = conn.blpop(queues)
|
||||||
|
else:
|
||||||
|
value = cls._dequeue_any(queues)
|
||||||
|
if value is None:
|
||||||
|
raise NoMoreWorkError('No more work.')
|
||||||
|
queue, msg = value
|
||||||
|
return (queue, msg)
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return self.name
|
return self.name
|
||||||
|
|
28
rq/worker.py
28
rq/worker.py
|
@ -10,13 +10,11 @@ except ImportError:
|
||||||
from pickle import loads, dumps
|
from pickle import loads, dumps
|
||||||
from .queue import Queue
|
from .queue import Queue
|
||||||
from .proxy import conn
|
from .proxy import conn
|
||||||
|
from .exceptions import NoMoreWorkError, NoQueueError
|
||||||
|
|
||||||
def iterable(x):
|
def iterable(x):
|
||||||
return hasattr(x, '__iter__')
|
return hasattr(x, '__iter__')
|
||||||
|
|
||||||
class NoQueueError(Exception): pass
|
|
||||||
class NoMoreWorkError(Exception): pass
|
|
||||||
|
|
||||||
class Worker(object):
|
class Worker(object):
|
||||||
def __init__(self, queues, rv_ttl=500):
|
def __init__(self, queues, rv_ttl=500):
|
||||||
if isinstance(queues, Queue):
|
if isinstance(queues, Queue):
|
||||||
|
@ -57,35 +55,13 @@ class Worker(object):
|
||||||
procname.setprocname('rq: %s' % (message,))
|
procname.setprocname('rq: %s' % (message,))
|
||||||
|
|
||||||
|
|
||||||
def multi_lpop(self, queues):
|
|
||||||
# Redis' BLPOP command takes multiple queue arguments, but LPOP can
|
|
||||||
# only take a single queue. Therefore, we need to loop over all
|
|
||||||
# queues manually, in order, and raise an exception is no more work
|
|
||||||
# is available
|
|
||||||
for queue in queues:
|
|
||||||
value = conn.lpop(queue)
|
|
||||||
if value is not None:
|
|
||||||
return (queue, value)
|
|
||||||
return None
|
|
||||||
|
|
||||||
def pop_next_job(self, blocking):
|
|
||||||
queues = self.queue_keys()
|
|
||||||
if blocking:
|
|
||||||
queue, msg = conn.blpop(queues)
|
|
||||||
else:
|
|
||||||
value = self.multi_lpop(queues)
|
|
||||||
if value is None:
|
|
||||||
raise NoMoreWorkError('No more work.')
|
|
||||||
queue, msg = value
|
|
||||||
return (queue, msg)
|
|
||||||
|
|
||||||
def _work(self, quit_when_done=False):
|
def _work(self, quit_when_done=False):
|
||||||
did_work = False
|
did_work = False
|
||||||
while True:
|
while True:
|
||||||
self.procline('Waiting on %s' % (', '.join(self.queue_names()),))
|
self.procline('Waiting on %s' % (', '.join(self.queue_names()),))
|
||||||
try:
|
try:
|
||||||
wait_for_job = not quit_when_done
|
wait_for_job = not quit_when_done
|
||||||
queue, msg = self.pop_next_job(wait_for_job)
|
queue, msg = Queue.dequeue_any(self.queues, wait_for_job)
|
||||||
did_work = True
|
did_work = True
|
||||||
except NoMoreWorkError:
|
except NoMoreWorkError:
|
||||||
break
|
break
|
||||||
|
|
Loading…
Reference in New Issue