django+celery+django-celery-beat+redis实现定时任务且动态配置和异步任务
一、在项目同级目录下新增celery.py、celeryconfig.py文件
二、项目下代码如下
__init__.py文件中新增如下代码
from __future__ import absolute_import, unicode_literals from .celery import app as celery_app # __all__ = ['celery_app']
broker和backend配置
# celery任务与结果 broker = "redis://localhost:6379/1" # 任务队列 backend = "redis://localhost:6379/2" # 结果存储
celeryconfig.py文件中设置celery的配置
from djangoAutoTest import settings from djangoAutoTest.env.common import broker from djangoAutoTest.env.common import backend # 设置结果存储 CELERY_RESULT_BACKEND = backend # 设置代理人broker BROKER_URL = broker # celery 的启动工作数量设置 CELERY_WORKER_CONCURRENCY = 20 # 任务预取功能，就是每个工作的进程／线程在获取任务的时候，会尽量多拿 n 个，以保证获取的通讯成本可以压缩。 CELERYD_PREFETCH_MULTIPLIER = 20 # 非常重要,有些情况下可以防止死锁 CELERYD_FORCE_EXECV = True # celery 的 worker 执行多少个任务后进行重启操作 CELERY_WORKER_MAX_TASKS_PER_CHILD = 100 # 禁用所有速度限制，如果网络资源有限，不建议开足马力。 CELERY_DISABLE_RATE_LIMITS = True CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler' DJANGO_CELERY_BEAT_TZ_AWARE = False CELERY_TIMEZONE = settings.TIME_ZONE CELERY_ENABLE_UTC = True
celery.py文件中初始化celery实例
from __future__ import absolute_import, unicode_literals import os from celery import Celery from celery.schedules import crontab from djangoAutoTest import settings, celeryconfig os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'djangoAutoTest.settings') app = Celery('djangoAutoTest') # 从settings中获取配置，所有配置需要带CELERY_前缀 app.config_from_object(celeryconfig) # # 定时任务 # app.conf.beat_schedule = { # # 进度汇报提醒 # 'notice_progress_report': { # 'task': 'haohan.tasks.notice_progress', # 设置是要将哪个任务进行定时 # 'schedule': crontab(hour=14, minute=16), # }, # } # 自动收集任务 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
三、在django的应用下创建tasks.py文件
tasks.py中写任务
from __future__ import absolute_import, unicode_literals from celery import shared_task from haohan.utils.jenkinsBuild import * from common.framework.log import Logger import datetime from common.framework.database import Mysql_DB from haohan.utils.sendFeishuMsg import SendFeiShuMsg from djangoAutoTest.env import common logger = Logger("submit_platform").get_logger() base_url = common.auto_platform_url # jenkins服务分支同步 @shared_task def jenkins_services_sync(): try: logger.info("开始jenkins服务同步") update_service_branch_data() logger.info("jenkins服务同步完成") except Exception as e: logger.error(f"jenkins服务同步失败：{e}") raise e
四、在settings中注册django_celery_beat
settings.py
INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'django_crontab', 'django_comment_migrate', 'haohan',
.... 'django_celery_beat', ]
五、生成django-celery-beat的数据表
python3 manage.py makemigrations
python3 manage.py migrate
六、管理定时任务（两种方式）
（1）、进入django的admin后台配置任务
python3 manage.py createsuperuser
（2）、自己开发界面去管理定时任务
新增任务：
附自定义任务管理代码：
后端：django
# -*- coding:utf-8 -*- import django import json import os import pytz from django_celery_beat import models as celery_models from django.http import JsonResponse, HttpResponse from celery import current_app from common.framework.log import Logger os.environ.setdefault("DJANGO_SETTINGS_MODULE", "djangoAutoTest.settings") django.setup() logger = Logger().get_logger() # 定时任务列表 def task_index(request): if request.method == "GET": all_crontab = list(celery_models.PeriodicTask.objects.values('id', 'name', 'task', 'args', 'kwargs', 'enabled', 'crontab')) for task in all_crontab: crontab_obj = celery_models.CrontabSchedule.objects.filter(id=task["crontab"]).first() task["run_time"] = '{0} {1} {2} {3} {4}'.format( crontab_obj.minute, crontab_obj.hour, crontab_obj.day_of_month, crontab_obj.month_of_year, crontab_obj.day_of_week) all_tasks_list = list(current_app.tasks.keys()) tasks_list = [] for i in all_tasks_list: if not str(i).startswith('celery'): tasks_list.append(i) return JsonResponse( {"code": 200, "data": {"all_crontab": list(all_crontab), "tasks_list": tasks_list}, "msg": "获取定时任务成功"}) # 获取任务信息 def get_task_info(request): if request.method == "POST": task_id = request.body["taskId"] task_obj = list( celery_models.PeriodicTask.objects.filter(id=task_id).values('id', 'name', 'task', 'args', 'kwargs', 'enabled', 'crontab')) crontab_obj = celery_models.CrontabSchedule.objects.filter(id=task_obj[0]["crontab"]).first() task_obj[0]["runTime"] = '{0} {1} {2} {3} {4}'.format( crontab_obj.minute, crontab_obj.hour, crontab_obj.day_of_month, crontab_obj.month_of_year, crontab_obj.day_of_week) return JsonResponse({"code": 200, "data": {"taskInfo": task_obj}, "msg": "获取定时任务成功"}) # 新增定时任务 def add_task(request): if request.method == "POST": try: task_name = request.body["name"] task_class = request.body["task"] enabled = request.body["enabled"] args = request.body["args"] kwargs = request.body["kwargs"] crontab_time = request.body["runTime"] run_time = str(crontab_time).strip().split(" ") if len(run_time) != 5: return JsonResponse({"code": 201, "msg": "定时任务时间格式错误"}) schedule, _ = celery_models.CrontabSchedule.objects.get_or_create(minute=run_time[0], hour=run_time[1], day_of_week=run_time[4], day_of_month=run_time[2], month_of_year=run_time[3], timezone=pytz.timezone('Asia/Shanghai')) task_obj = celery_models.PeriodicTask.objects.create(crontab=schedule, name=task_name, args=args, kwargs=kwargs, task=task_class, enabled=enabled) task_obj.save() celery_models.PeriodicTasks.changed(task_obj) return JsonResponse({"code": 200, "msg": "添加定时任务成功"}) except Exception as e: logger.error(f"添加定时任务失败：{e}") return JsonResponse({"code": 201, "msg": "添加定时任务失败"}) # 编辑任务 def edit_task(request): if request.method == "POST": try: crontab_time = request.body["runTime"] run_time = str(crontab_time).strip().split(" ") if len(run_time) != 5: return JsonResponse({"code": 201, "msg": "定时任务时间格式错误"}) else: celery_models.PeriodicTask.objects.filter(id=request.body["id"]).update(name=request.body["name"], task=request.body["task"], args=request.body["args"], kwargs=request.body["kwargs"], enabled=request.body["enabled"]) celery_models.CrontabSchedule.objects.filter(id=request.body["crontab"]).update( minute=run_time[0], hour=run_time[1], day_of_week=run_time[4], day_of_month=run_time[2], month_of_year=run_time[3], ) task_obj = celery_models.PeriodicTask.objects.get(id=request.body["id"]) celery_models.PeriodicTasks.changed(task_obj) return JsonResponse({"code": 200, "msg": "编辑任务成功"}) except Exception as e: logger.error(f"编辑任务失败：{e}") return JsonResponse({"code": 201, "msg": "编辑任务失败"}) # 删除任务 def delete_task(request): if request.method == "POST": try: task_id = request.body["id"] task_obj = celery_models.PeriodicTask.objects.get(id=task_id) task_obj.crontab.delete() task_obj.delete() return JsonResponse({"code": 200, "msg": "任务删除成功"}) except Exception as e: logger.error(f"删除任务失败：{e}") return JsonResponse({"code": 201, "msg": "任务删除失败"})
前端代码：vue
<template> <div class="fillcontain"> <div class="col"></div> <div class="homeTitle">定时任务</div> <div class="hr_bottom"></div> <div style="height: 45px; position: relative"> <el-button type="primary" @click="addCrontab" class="priBtnSty">新增任务 </el-button> </div> <el-container> <el-table :data="tableData"> <el-table-column prop="id" header-align="left" label="序号" min-width="15%" align="left"> </el-table-column> <el-table-column prop="name" header-align="left" label="任务名称" show-overflow-tooltip min-width="50%" align="left"> </el-table-column> <el-table-column prop="task" header-align="left" min-width="50%" show-overflow-tooltip label="方法名称" align="left"> </el-table-column> <el-table-column prop="args" header-align="left" min-width="50%" show-overflow-tooltip label="位置参数" align="left"> </el-table-column> <el-table-column prop="kwargs" header-align="left" min-width="50%" show-overflow-tooltip label="关键字参数" align="left"> </el-table-column> <el-table-column prop="run_time" header-align="left" min-width="30%" align="left"> <template slot-scope="scope" slot="header"> <el-tooltip effect="dark" :content="'格式含义：' + meaning" placement="top"> <span>执行时间 <i class="el-icon-info"></i></span> </el-tooltip> </template> </el-table-column> <el-table-column prop="enabled" header-align="left" label="启用" min-width="15%" align="left"> <template slot-scope="scope"> <span v-if="scope.row.enabled" style="color: green; font-weight: bolder">是</span> <span v-if="!scope.row.enabled" style="color: red; font-weight: bolder">否</span> </template> </el-table-column> <el-table-column header-align="right" align="right" label="操作" width="150" > <template slot-scope="scope"> <el-button type="primary" @click="editCrontab(scope.row)" class="btnSty">编辑 </el-button> <el-button type="primary" @click="deleteCrontab(scope.row)" class="btnSty">删除 </el-button> </template> </el-table-column> </el-table> </el-container> <pagiationCpn :pageTotal="pagiations.total" :cur-page="processData.page" @handleCurrentChange="handleCurrentChange" @handleSizeChange="handleSizeChange" > </pagiationCpn> <tasksDialogCpn :tasksDialog="tasksDialog" v-if="tasksDialog" :taskId="taskId" @closeDialog="closeDialog" :title="title" :tasksList="tasksList"></tasksDialogCpn> </div> </template> <script> import pagiationCpn from "cps/pagination/index"; import tasksDialogCpn from "@/page/tasksManage/components/tasksDialog"; import {axiosRequest} from "@/utils/common"; export default { name: "index", data() { return { tableData: [], pagiations: { total: 0 }, processData: { page: 1, limit: 10, }, meaning: "分/时/日/月/周几", tasksDialog: false, taskId: "", title: "", tasksList: [], } }, components: { pagiationCpn, tasksDialogCpn, }, mounted() { this.getCrontabList() }, methods: { getCrontabList() { axiosRequest('/haohan/task_index/', "get").then(res => { if (res.code === 200) { this.tableData = res.data.all_crontab this.tasksList = res.data.tasks_list } }) }, addCrontab() { this.tasksDialog = true this.title = "新增任务" }, editCrontab(row) { this.tasksDialog = true this.title = "编辑任务" this.taskId = row.id }, deleteCrontab(row) { this.$confirm('确认删除此任务,是否继续?', '提示', { confirmButtonText: '确定', cancelButtonText: '取消', type: 'warning' }).then(() => { this.taskId = row.id const params = Object.assign({}, {"id": this.taskId}) axiosRequest('/haohan/delete_task/', 'post', params).then(res => { if (res.code === 200){ this.$message.success(res.msg) this.getCrontabList() }else { this.$message.error(res.msg) } }).catch(() => { this.$message.info("取消删除") }) }) }, // 上下分页 handleCurrentChange(val) { this.processData.page = val; this.getCrontabList() }, // 每页显示多少条 handleSizeChange(val) { this.processData.limit = val; this.getCrontabList() }, closeDialog() { this.tasksDialog = false }, } } </script> <style lang="less"> @import "../../style/base.less"; </style>
新增任务的dialog
<template> <div> <!-- 添加或修改定时任务对话框 --> <el-dialog :title="title" :visible.sync="isVisible" :close-on-click-modal='false' :close-on-press-escape='false' :modal-append-to-body="false" @close="closeDialog" > <el-form ref="taskForm" :model="taskForm" :rules="rules" label-width="150px"> <el-form-item label="任务名称" prop="name"> <el-col :span="14"> <el-input v-model="taskForm.name" placeholder="请输入任务名称" style="width: 100%"/> </el-col> </el-form-item> <el-form-item label="执行方法" prop="task"> <el-col :span="14"> <el-select v-model="taskForm.task" placeholder="请选择执行的方法" filterable clearable style="width: 100%"> <el-option v-for="item in tasksList" :key="item.index" :value="item" :label="item"></el-option> </el-select> </el-col> </el-form-item> <el-form-item label="位置参数"> <el-col :span="14"> <el-input v-model="taskForm.args" placeholder="请输入位置参数(Example: ['arg1', 'arg2'])" style="width: 100%"/> <span style="font-size: 12px; color: #9a9999">Example: ["arg1", "arg2"]</span> </el-col> </el-form-item> <el-form-item label="关键字参数"> <el-col :span="14"> <el-input v-model="taskForm.kwargs" placeholder="请输入关键字参数(Example: {'argument': 'value'})" style="width: 100%"/> <span style="font-size: 12px; color: #9a9999">Example: {"argument": "value"}</span> </el-col> </el-form-item> <el-form-item label="执行时间" prop="runTime"> <el-col :span="14"> <el-input v-model="taskForm.runTime" placeholder="请输入定时任务执行时间" style="width: 100%"/> <img src="../components/c2.png" style="width: 100%;"> </el-col> </el-form-item> <el-form-item label="是否启用" prop="enabled"> <el-col :span="14"> <el-checkbox v-model="taskForm.enabled"></el-checkbox> </el-col> </el-form-item> </el-form> <div slot="footer" class="dialog-footer"> <el-button @click="isVisible = false">取 消</el-button> <el-button type="primary" @click="submitTaskForm('taskForm')">确 定</el-button> </div> </el-dialog> </div> </template> <script> import {axiosRequest} from "@/utils/common"; export default { name: "tasksDialog", data(){ return { isVisible: '', taskForm: { name: "", task: "", runTime: "", enabled: true, args: '[]', kwargs: '{}', }, rules: { name: [{required: true, message: '任务名称不能为空', trigger: 'blur'}], task: [{required: true, message: '执行方法不能为空', trigger: 'blur'}], runTime: [{required: true, message: '执行时间不能为空', trigger: 'blur'}], }, } }, props: ['taskId', 'title', 'tasksList'], mounted(){ if (this.title === "编辑任务") { this.getTaskInfo() } }, methods: { submitTaskForm(form){ this.$refs[form].validate((valid) => { if (valid) { if (this.taskForm.id){ axiosRequest('/haohan/edit_task/', "post", this.taskForm).then(res => { if (res.code === 200){ this.$message.success(res.msg) this.isVisible = false this.$parent.getCrontabList() }else { this.$message.error(res.msg) } }) }else { axiosRequest("/haohan/add_task/", "post", this.taskForm).then(res => { if (res.code === 200) { this.$message.success(res.msg) this.isVisible = false this.$parent.getCrontabList() }else { this.$message.error(res.msg) } }) } } }) }, getTaskInfo(){ axiosRequest('/haohan/get_task_info/', "post", {"taskId": this.taskId}).then(res => { if (res.code === 200) { this.taskForm = res.data.taskInfo[0] } }) }, closeDialog(){ this.$emit('closeDialog'); }, } } </script> <style scoped> </style>
七、启动celery和beat
# 启动任务 worker 并指定日志输出文件 celery -A djangoAutoTest worker -l INFO --logfile=./common/logs/submit_platform.log # 启动定时器触发 beat 并指定日志输出文件 celery -A djangoAutoTest beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler --logfile=./common/logs/submit_platform.log
建议：
服务器上可以让运维将启动命令加在jenkins构建的脚本中
RUN echo "nohup celery -A djangoAutoTest worker -l INFO --logfile=./common/logs/submit_platform.log &" >> /opt/${ServiceName}/start.sh RUN echo "nohup celery -A djangoAutoTest beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler --logfile=./common/logs/submit_platform.log &" >> /opt/${ServiceName}/start.sh RUN echo "python3 manage.py runserver 0.0.0.0:sport" >> /opt/${ServiceName}/start.sh
也可以自己去服务器上执行后台运行
nohup celery -A djangoAutoTest worker -l INFO --logfile=./common/logs/submit_platform.log &
nohup celery -A djangoAutoTest beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler --logfile=./common/logs/submit_platform.log &
