django中celery的使用

创建django项目

$ django-admin startproject proj
$ cd proj
$ tree
.
├── manage.py
└── proj
    ├── __init__.py
    ├── asgi.py
    ├── settings.py
    ├── urls.py
    └── wsgi.py

项目引入celery

创建一个新的 proj/proj/celery.py模块，定义celery实例:

文件：proj/proj/celery.py

import os

from celery import Celery

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings')

app = Celery('proj')

# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
#   should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')

# Load task modules from all registered Django app configs.
# 这里会加载多有的task
app.autodiscover_tasks()

将proj/proj/init.py 中导入模块，这样可以确保在django启动的时候加载应用。

文件 proj/proj/init.py

from .celery import app as celery_app
__all__=('celery_app',)

文件 proj/proj/settings.py

import os

# ^^^ The above is required if you want to import from the celery
# library.  If you don't have this then `from celery.schedules import`
# becomes `proj.celery.schedules` in Python 2.x since it allows
# for relative imports by default.

# celery设置
# 1. backend_url
CELERY_BROKER_URL = 'amqp://guest:guest@localhost'

#: Only add pickle to this list if your broker is secured
#: from unwanted access (see userguide/security.html)
CELERY_ACCEPT_CONTENT = ['json']
# celery backend 地址
CELERY_RESULT_BACKEND = 'db+sqlite:///results.sqlite'
# celery backend 序列化类型
CELERY_TASK_SERIALIZER = 'json'


"""
Django settings for proj project.
Generated by 'django-admin startproject' using Django 2.2.1.
For more information on this file, see
https://docs.djangoproject.com/en/2.2/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/2.2/ref/settings/
"""


# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'l!t+dmzf97rt9s*yrsux1py_1@odvz1szr&6&m!f@-nxq6k%%p'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []


# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'demoapp',
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'proj.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'proj.wsgi.application'


# Database
# https://docs.djangoproject.com/en/2.2/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}


# Password validation
# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]


# Internationalization
# https://docs.djangoproject.com/en/2.2/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True


# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.2/howto/static-files/

STATIC_URL = '/static/'

tasks文件项目中的位置

tasks文件位置一般有以下几种方案

方案一: 放到对应到app目录下面

├── app
│   ├── __init__.py
│   ├── admin.py
│   ├── apps.py
│   ├── migrations
│   │   └── __init__.py
│   ├── models.py
│   ├── tasks.py   # 创建一个task文件， 这里是worker相关代码
│   ├── tests.py
│   └── views.py
├── manage.py
└── proj
    ├── __init__.py
    ├── asgi.py
    ├── settings.py
    ├── celery.py  # celery 声明文件
    ├── urls.py
    └── wsgi.py

方案二: 单独创建一个task路径

├── celery_tasks
│   ├── __init__.py
│   ├── app01
│   │   ├── __init__.py
│   │   └── tasks.py   # app01相关tasks代码
│   ├── app02
│   │   ├── __init__.py
│   │   └── tasks.py   # app02相关tasks代码
│   └── app03
│       ├── __init__.py
│       └── tasks.py   # app03相关tasks代码
├── manage.py
└── proj
    ├── __init__.py
    ├── asgi.py
    ├── settings.py
    ├── celery.py # celery声明文件
    ├── urls.py
    └── wsgi.py

方案三: 直接在项目目录下创建

├── manage.py
├── proj
│   ├── __init__.py
│   ├── asgi.py
│   ├── settings.py
│   ├── celery.py # celery 声明文件
│   ├── urls.py
│   └── wsgi.py
└── tasks.py # task任务文件

编辑tasks文件

这里我们按照方案2的结构进行处理，目录为如下结构
假设 app1 为算数运算的服务， app2 为邮件服务

- app1/  
    - tasks.py   # 算数运算
    - models.py
- app2/
    - tasks.py   # 发送邮件
    - models.py

文件： app1/tasks.py

from celery import app
from app1.models import Widget


@shared_task
def add(x, y):
    return x + y


@app.task
def mul(x, y):
    return x * y


@app.task
def xsum(numbers):
    return sum(numbers)


@app.task
def count_widgets():
    return Widget.objects.count()


@app.task
def rename_widget(widget_id, name):
    w = Widget.objects.get(id=widget_id)
    w.name = name
    w.save()

文件 app2/tasks.py

from celery import app
import time
@app.task
def sendmail():
    print('sendmail mail to yunweigo')
    time.sleep(5.0)
    print('mail sent success')
    

通过数据库保存结果（myql关系数据库或者redis非关系数据库）

通过数据库保存 backends 存储数据

django-celery-results - Using the Django ORM/Cache as a result backend¶
pip install django-celery-results
INSTALLED_APPS = (
    ...,
    'django_celery_results',
)
$ python manage.py migrate django_celery_results

Configure Celery to use the django-celery-results backend.

Assuming you are using Django’s settings.py to also configure Celery, add the following settings:

CELERY_RESULT_BACKEND = 'django-db'
For the cache backend you can use:

CELERY_RESULT_BACKEND = 'django-cache'
We can also use the cache defined in the CACHES setting in django.

# celery setting.
CELERY_CACHE_BACKEND = 'default'

# django setting.
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.db.DatabaseCache',
        'LOCATION': 'my_cache_table',
    }
}

启动过服务

$ celery -A proj worker -l INFO

 

获取backends 中结果

class GetResultTask(View):
    # 获取任务的结果

    def get(self, request):
　　　　 task_id = request.query_params.get("task_id")
        task_result = AsyncResult(task_id)  'ccbbb7a5-fdb1-4d46-87ad-a8be7d7be674')  # 这个ID可以设置成动态的
        print(task_result.result)  # 获取任务return返回结果
        print(task_result.status)  # 获取任务执行结果状态 
        print(task_result.successful()) # 是否完成
        return Response(str(task_result.result))

 因此，在获取异步结果中常见的命令：

from celery.result import AsyncResult 

task_id # 生产者返回的任务id

res = AsyncResult(task_id)

state = res.ready() # 判断是否完成
res.successful() # 判断是否成功
res.result  # 获取结果

 

 

注意事项：

（1）、要启动定时任务，先要启动异步任务；

（2）、windows下执行运行Worker，任务不执行，必需使用-P eventlet启动，同时连接Redis必需使用IP地址，不能使用localhost；

（3）、启动Worker时必需和消息队列保持连通，修改任务函数后，必需重启Worker；

（4）、启动Worker时可以指定消息队列，但是必需在配置文件中配置，或调用任务时指定队列名；

（5）、如果都是使用默认队列celery，启动Worker时可能会收到大量历史任务并进行处理；

（6）、定时任务celery beat如果没有及时关闭，会一直按要求发送异步任务，产生大量历史遗留任务。

 

celery的常用命令：

帮助文档 多看看 
celery --help 
#常规启动Worker 
celery -A tasks worker --loglevel=INFO 
#Windows下启动Worker 
celery -A tasks worker --loglevel=INFO -P eventlet 
#关闭Worker 
Ctrl + C
,可能需要连续按 

#启动Beat程序 可以帮我们定时发送任务到消息队列 
celery -A tasks beat --loglevel=INFO 

 

 

 

参考：https://blog.csdn.net/yunweigo/article/details/115267014，  https://zhuanlan.zhihu.com/p/587412153?utm_id=0