| apscheduler, gunicorn

Run Apscheduler with Gunicorn in More Than One Machine

Shot Cat in Dot with Code

If you use apscheduler in your web application, and start by gunicorn on many machines. It must be more than one running worker at the same time.

To handle this, we can use redis to control the workers. At the beginning of worker’s function, checking the redis key. If key exist, return.

Demo Code

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
import time
import random
from uuid import uuid4

import redis


redis = redis.Redis(host='127.0.0.1', port=6379)

worker_key = 'worker_key_1'
worker_max_num = 3


class WorkerLock:

    def __init__(key: str, max_num: int = 1, expire_time: int = 60):
        self.key = key
        self.max_num = num
        self.expire_time = expire_time
        self.worker_id = self.get()

    def get(self):
        """get worker id

        """
        # sleep random (0-1)second, to avoid more than one worker get and incr key at the same time
        time.sleep(random.random())
        val = redis.get(self.key)

        if val and int(val) >= self.max_num:
            return None

        redis.incr(self.key)
        redis.expire(self.key, self.expire_time)
        worker_id = uuid4().hex
        return worker_id
    
    def release():
        """release worker lock

        """
        if self.worker_id is None:
            return
        
        if self.max_num = 1:
            redis.delete(self.key)
            return
        redis.decr(self.key)
        return


def worker_function():
    """the function being called in apscheduler

    """
    w_lock = WorkerLock(key=worker_key, max_num=worker_max_num)
    if w_lock.worker_id is None:
        return
    # start do the job
    print('I got the key.')
    w_lock.release()
    return