mongodb总用方法总结
mongodb总用方法总结
pymongo存在更新不存在插入
{'$set': data, '$setOnInsert': {'create_time': datetime.datetime.now()}}
$setOnInsert
里面使用的字段是数据不存在的时候才插入的,存在就不动了,只插入$set
里面指定的。
另外$setOnInsert
里面使用的字段不能在$set
里面再次出现
upsert=True代表的是不存在就插入。
res_content = collection_v2_pongo_keyword.find(filter_, no_cursor_timeout=True, batch_size=5000).allow_disk_use(True)
mongodb中如何删除字段
删除mongo表中字段的方法:
1
|
db.collection.update({},{ $unset :{ "需要删除的字段" : "" }},false,true) |
update命令格式:
1
|
db.collection.update(criteria,objNew,upsert,multi) |
参数说明:
criteria:查询条件
objNew:update对象和一些更新操作符
upsert:如果不存在update的记录,是否插入objNew这个新的文档,true为插入,默认为false,不插入。
multi:默认是false,只更新找到的第一条记录。如果为true,把按条件查询出来的记录全部更新。
collection_v2_pongo_keyword.find(filter_, no_cursor_timeout=True, batch_size=5000).allow_disk_use(True)
# 删除发布list中指定id 的数据. 删除数组对象中指定的数据
"$pull": {"publishlist": {"id": publishlist_id}}
// aggregate db.lily_k_infomation.aggregate([ {$project: { // 字符串拼接 _id:{"$toString":"$_id"}, infoid: "$id", keyword: 1, path: { "$concat" : ["/detail", "$urlpath"]}, // 修改字段名称, 必须引号包裹 // aid: "$allianceid",
// 计算数字长度
aids : {"$size": "$aids"}
} }, // {$match: {}}, ])
db.pongo_wordgroup.aggregate([ { '$match': {'owner': 'rshy'} }, { '$project': { // 不能出现为0的 _id可以 其余字段需要手动写出 //'state': 0, "_id": 0, "name": 1, "id": {"$toString": "$_id"}, 'itemnum': {'$size': '$items'}, // "name": 1, } }, ])
db.v2_pongo_keyword.aggregate([
{"$match": {"industryid2": "10421082"}},
{
"$project": {
"_id": 0,
"id": {"$toObjectId": "$wordgroupid"},
"orgid": 1,
"keyword": 1,
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"wordgroupid": 1,
"baiduindex": 1,
// "createinfo": 1,
}
},
{
"$lookup": {
"from": "v2_pongo_wordgroup",
"localField": "id",
"foreignField": "_id",
"as": "inventory_docs"
},
},
{
"$project": {
"_id": 0,
"orgid": 1,
"keyword": 1,
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"wordgroupid": 1,
"baiduindex": 1,
"createinfo": "$inventory_docs.createinfo",
}
},
], { allowDiskUse: true})
// 查询 db.lily_infomation.find({"links": null}) // 修改字段名称 db.getCollection("lily_infomation").update({}, {$rename: {"specialtopics": "topic"}}, {multi: true})
在MongoDB操作的时候,我们可能会遇到查询【字段对应的数组中包含某个值】的数据,查询语句如下,假设表名为user。
示例1
- 数据
{ id: 1, state_arr: [ "123", "456", "789" ], date: "2019-09-02" }
- 查询语句
db.user.find({state_arr:{$elemMatch:{$eq:"123"}}})
示例2
- 数据
{ id: 1, state_arr: [ { id: 1, name: "hhh" }, { id: 2, name: "www" } ], date: "2019-09-02" }
db.lily_infomation.aggregate([
{ '$match': { "tags": {"$elemMatch" : {"id": {"$in" : [11]} } } } },
{ '$sort': { datecreated: -1 } },
{ '$limit': 20 },
])
db.lily_infomation.aggregate([
{ '$match': { "tags": {"$all" : [{"id":1, "name": 3}, {"id":2, "name": 2}] } } },
{ '$sort': { datecreated: -1 } },
{ '$limit': 20 },
])
- 查询语句
db.user.find({state_arr:{$elemMatch:{id:1}}})
// forEach db.getCollection("lily_infomation").find({}).forEach(function(item){ var Id = item._id; var infoid = item.infoid; print(Id, infoid) var li = [] db.lily_specialtopicinformation_new.find( { InfoId: infoid }).forEach(function(item_label){ var SpecialTopicId = item_label.SpecialTopicId; //print(SpecialTopicId); db.lily_specialtopicalliance.find( { id: SpecialTopicId }).forEach(function(item_label_){ var li_id = item_label_.id; var li_name = item_label_.words; //print(li_id, li_name) li.push({"id": li_id, "name": li_name, "urlpath": "/peixun/k_" + li_id + ".html" }) }); // print(li); }); db.getCollection("lily_infomation").update( { _id: Id }, { $set: { topic: li } }); }); // 创建索引 db.publish_tp.createIndex({"uuid": 1}) // or db.syd.find({"$or": [{"lable1": "综合"}, {"lable1": "少儿"}]}) // group db.syd.aggregate([{$group : {_id : "$lable1", total : {$sum : 1}}}]) db.fanwen_c.aggregate([ { $group : {_id : "$status", num_tutorial : {$sum : 1}}, }, { $sort: {_id: 1} } ]) // regex db.publish.find({"title": {"$regex":"_"}, "status": 22}) // $lt $gt db.publish_tp.find({"status": 1, "dateCreated": {"$lt": "2021-06-01"}}).sort({"dateCreated": -1}).count() // count db.asx.find().count() // $rename 修改字段名称 db.getCollection("lily_infomation").update({}, {$rename: {"specialtopics(旧字段名)": "topic(新字段名)"}}, {multi: true}) // // 数组去重 function function unique (arr) { return Array.from(new Set(arr)) } db.getCollection("lily_teacher").update( { _id: item._id }, { $set: { labels: unique(new_labelid) } });
将一个对象数组挨个塞入数组中
params = {
"$set": temp,
"$addToSet": {"labellist": {"$each": labellist}}
}
// $lookup
var db_name = await getDBName(allianceId); var dbo = db.db(db_name); var filter = { labelid1:labelid1, labelid2:labelid2, labelid3:labelid3 }; var optional = [ { $match: filter }, { $lookup: { from: "lily_areacode", localField: "areacode", foreignField: "areacode", as: "docs" } }, { $unwind: "$docs" }, { $project: { _id: 0, province: "$docs.province", city: "$docs.city", longitude: "$docs.longitude", latitude: "$docs.latitude" } } ]; var allow = { allowDiskUse: true}; console.log(optional); var result = await dbo.collection("lily_tdk_list").aggregate(optional,allow).toArray();
var optional = [ { $match: filter }, { $sort: {datecreated: -1} }, { $limit: size }, { $project: { 'allianceid':1, "infoid":1, "title": 1, "datecreated": 1, "dateupdated": 1, "content": { $function: { body: 'function(name) { \ let d = name[0] ; \ let content = d.content; \ content = content.replace(/<[^>]+>/g,""); \ return content.substr(0, 150) ; \ }', args: [ "$contents" ], lang: "js" } }, // 新增回复了量字段 "reply": { "$size": "$contents"}, "urlpath": 1, "imagepath":1, "labelid1": 1, "labelid2": 1, "labelid3": 1, "labelname1": 1, "labelname2": 1, "labelname3": 1, "teachershowid":1, "teachernickname": 1, "teacherheadimage":1, "infotype": 1, "show": "$view.show" } }];
var optional = [ { $limit: page*size },{ $skip: (page-1)*size }, { $project: { '_id':0, 'infoid':1, "title": 1, "datecreated": 1, "urlpath": 1 } }];
var optional = [ { $match: filter }, {"$unwind":"$info"}, { $limit: 100 }, { $match: filter2 }, { $project: { kemu: 1, period: 1, knowledge:{ $function:{ body:'function(maps){\ for(var i in maps){\ if(maps[i].indexOf("知识点")!=-1){\ return maps[i];\ } \ }\ }', args: [ "$info.zsd" ], lang: "js" } } } }];
var optional = [ { $match: filter }, { $project: { '_id':1, "keyword":1, "urlpath": 1, "labelid1": 1, "labelid2": 1, } }, { $sample: { size: size } } ];
pipeline = [ {"$match": filter_}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "name": 1, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "itemnum": {f"$size": f"$items"}, "ii": { "$function": { "body": 'function(name) { \ return name + "------"; \ }', "args": ["$name"], "lang": "js" } } } }, ] print(pipeline) res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
// 对数组中全部元素进行修改
db.getCollection("pongo_content").update({"templateid": "simian-vpc-96232940694139658ae408deec27ae51", "titles": {"$elemMatch": {"id": "simian_vpt-527d8d3cad874dc598de0d91e1fa99fe"}}}, { "$set": { "titles.$.status": 1, } } )
// 对数组中指定元素进行修改 db.getCollection("pongo_content").update({ "templateid": "simian-vpc-96232940694139658ae408deec27ae51", }, { "$set": { "titles.$[].status": 1, } }, // { "multi": true } )
// 对数组中满足条件元素进行修改 db.getCollection("pongo_content").update({ "templateid": "simian-vpc-96232940694139658ae408deec27ae51", }, { "$set": { "titles.$[elem].status": 2, } }, { "multi": true, arrayFilters: [ { "elem.status": 222222 } ] } )
参考:https://docs.mongodb.com/manual/reference/operator/update/positional-filtered/
【MongoDB】查询字段对应的数组中包含某个值 在MongoDB操作的时候,我们可能会遇到查询【字段对应的数组中包含某个值】的数据,查询语句如下,假设表名为user。 示例1 数据 { id: 1, state_arr: [ "123", "456", "789" ], date: "2019-09-02" } 查询语句 db.user.find({state_arr:{$elemMatch:{$eq:"123"}}}) 示例2 数据 { id: 1, state_arr: [ { id: 1, name: "hhh" }, { id: 2, name: "www" } ], date: "2019-09-02" } 查询语句 db.user.find({state_arr:{$elemMatch:{id:1}}})
if aid: filter_.update({"publishlist": {"$elemMatch": {"aid": str(aid)}}}) pipeline = [ {"$match": filter_}, {"$sort": {"keyword": 1}}, {"$limit": page * size}, {"$skip": (page - 1) * size}, {"$unwind": {"path": "$publishlist", "preserveNullAndEmptyArrays": True}}, # { # "$group": { # "_id": "$publishlist.aid", # }, # }, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "wordgroup": 1, "wordgroupid": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "keyword": 1, "area": 1, "orgid": 1, "orgname": 1, "baiduindex": 1, if aid: filter_.update({"publishlist": {"$elemMatch": {"aid": str(aid)}}}) pipeline = [ {"$match": filter_}, {"$sort": {"keyword": 1}}, {"$limit": page * size}, {"$skip": (page - 1) * size}, {"$unwind": {"path": "$publishlist", "preserveNullAndEmptyArrays": True}}, # { # "$group": { # "_id": "$publishlist.aid", # }, # }, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "wordgroup": 1, "wordgroupid": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "keyword": 1, "area": 1, "orgid": 1, "orgname": 1, "baiduindex": 1, "paid": "$publishlist.id", "aid": "$publishlist.aid", "aname": "$publishlist.aname", "labelname1": "$publishlist.labelname1", "labelname2": "$publishlist.labelname2", "labelname3": "$publishlist.labelname3", } }, ] if aid: pipeline.insert(-1, {"$match": {"publishlist.aid": str(aid)}},) "paid": "$publishlist.id", "aid": "$publishlist.aid", "aname": "$publishlist.aname", "labelname1": "$publishlist.labelname1", "labelname2": "$publishlist.labelname2", "labelname3": "$publishlist.labelname3", } }, ] if aid: pipeline.insert(-1, {"$match": {"publishlist.aid": str(aid)}},)
# _*_ coding: utf-8 _*_ from django.shortcuts import render, HttpResponse from django.http import JsonResponse from django.views.decorators.csrf import csrf_exempt, csrf_protect # Create your views here. import os import oss2 import uuid import sys import ast import time import json import redis import hashlib import requests import random from tools.mysql_client import LogMysqlApe, LogMysql, LogMysqlApeT, LogMysqlAlliance from tools.excel_client import read_excel from upload_pro.settings import logger from urllib.parse import urlparse, unquote from django.conf import settings from tools.mongodb_client import MongoDBClient from bson import ObjectId from bson import json_util from bson.json_util import dumps import pymongo from pymongo.errors import DuplicateKeyError from tools.format_time import dateRange def get_uuid(): res = str(uuid.uuid4()) UUID = ''.join(res.split('-')) return UUID def md5_me(key): md5 = hashlib.md5() md5.update(str(key).encode('utf-8')) value = md5.hexdigest() return value # 把格式化时间转换成时间戳 def str_to_timestamp(str_time=None, format='%Y-%m-%d %H:%M:%S'): if str_time: time_tuple = time.strptime(str_time, format) # 把格式化好的时间转换成元祖 result = time.mktime(time_tuple) # 把时间元祖转换成时间戳 return int(result) return int(time.time()) # 把时间戳转换成格式化 def timestamp_to_str(timestamp=None, format='%Y-%m-%d %H:%M:%S'): if timestamp: time_tuple = time.localtime(timestamp) # 把时间戳转换成时间元祖 result = time.strftime(format, time_tuple) # 把时间元祖转换成格式化好的时间 return result else: return time.strftime(format) # redis主库 class Redis(object): conn = None def __init__(self): # poll = redis.ConnectionPool(host='47.94.218.171', port=6379, db=11, password='root1234@A') # 生产 2 为测试 3为线上 poll = redis.ConnectionPool(host='192.168.5.219', port=6379, db=settings.APE_REDIS_DB, password='root1234@A', decode_responses=True) # 本地测试 # poll = redis.ConnectionPool(host='192.168.10.10', port=7000, db=11) self.conn = redis.Redis(connection_pool=poll) ape_db = settings.APE_DB db = settings.PONGO_DB pongo_wordgroup = "pongo_wordgroup" pongo_content = "pongo_content" pongo_category = "pongo_category" pongo_person = "pongo_person" pongo_industry = "pongo_industry" pongo_keyword = "pongo_keyword" pongo_title = "pongo_title" pongo_industry_ids = "pongo_industry_ids" pongo_tdk_title = "pongo_tdk_title" pongo_alliance = "pongo_alliance" pongo_publish_task = "pongo_publish_task" simian_viewpoint_table = "simian_viewpoint_table" simian_wordgroup_table = "simian_wordgroup_table" simian_vptitle_table = "simian_vptitle_table" simian_wordgroup_viewpoint_table = "simian_wordgroup_viewpoint_table" simian_vpcontent_table = "simian_vpcontent_table" pongo_produce_vpc_vpt = "pongo.list.produce_vpc_vpt" CODE_DICT = { "0000": "成功", "1000": "不存在标题id", "1010": "操作无权限", "1011": "此一级行业已经分配二级行业,不能删除!", "1012": "行业已分配角色,不能删除!", "1013": "无此机构数据!", "1014": "titles字段数据有误!", "1015": "该词分类名称已存在!", "1016": "没有机构,故没有数据!", "1017": "没有人物,故没有数据!", "1018": "此关键词已插入!", "1019": "需要带行业进行筛选!", "1020": "传入数据为空!", "1021": "请选择行业!", "1022": "标题重复!", "1023": "不满足生成替换文的条件!", "1024": "不满足删除条件!", "1025": "没有权限!", "1026": "数据有误!", } # 词组列表 @csrf_exempt def getWordgroupList(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_wordgroup] filter_ = {} if ownerid: filter_ = {"ownerid": ownerid} if owner: filter_ = {"owner": owner} pipeline = [ {"$match": filter_}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "name": 1, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "itemnum": {f"$size": f"$items"}, "ii": { "$function": { "body": 'function(name) { \ return name + "------"; \ }', "args": ["$name"], "lang": "js" } } } }, ] print(pipeline) res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] if res: data = list(res) return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 词组详情 @csrf_exempt def getWordgroupInfo(request): _datecreated = timestamp_to_str() try: if request.method == "GET": wordgroupid = request.GET.get('wordgroupid', '') name = request.GET.get('wordgroupname', '') items = request.GET.get('items', []) owner = request.GET.get('owner', '') ownerid = str(request.GET.get('ownerid', '')) orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') industry1 = request.GET.get('industry1', '') industry2 = request.GET.get('industry2', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_wordgroup] filter_ = {"orgid": orgid, "name": name} params = { "_id": 0, "id": {f"$toString": f"$_id"}, "name": 1, "items": 1, "owner": 1, "ownerid": 1, } print(filter_) # res = collection.find_one(filter_, no_cursor_timeout=True) res = collection.find_one(filter_, params, no_cursor_timeout=True) data = {} if res: data = res return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 新增词组 @csrf_exempt def newWordgroupInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) name = get_data.get('name', '') items = get_data.get('items', []) owner = get_data.get('owner', '') ownerid = str(get_data.get('ownerid', '')) orgid = get_data.get('orgid', '') orgname = get_data.get('orgname', '') industry1 = get_data.get('industry1', '') industryid1 = get_data.get('industryid1', '') industry2 = get_data.get('industry2', '') industryid2 = get_data.get('industryid2', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_wordgroup] filter_ = {"orgid": orgid, "name": name} res_filter = collection.find_one(filter_) if res_filter: for item in items: params = { "$addToSet": {"items": item}, "$set": { "dateupdated": TIME, }, } collection.update_one(filter_, params) else: data = { "name": name, "industry1": industry1, "industryid1": industryid1, "industry2": industry2, "industryid2": industryid2, "items": items, "orgid": orgid, "orgname": orgname, "owner": owner, "ownerid": ownerid, "datecreated": TIME, "dateupdated": TIME, "sort": 10, "state": 0 } collection.insert_one(data) return JsonResponse({ "code": "0000", "message": "成功", "data": "", "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 修改词组 @csrf_exempt def updateWordgroupInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) name = get_data.get('name', '') items = get_data.get('items', []) wordgroupid = get_data.get('wordgroupid', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_wordgroup] params = { "$set": { "name": name, "items": items, "dateupdated": TIME, }, } filter_ = {"_id": ObjectId(wordgroupid)} res = collection.update_one(filter_, params) return JsonResponse({ "code": "0000", "message": "成功", "data": wordgroupid, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 删除词组 @csrf_exempt def deleteWordgroupInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) name = get_data.get('name', '') items = get_data.get('items', []) wordgroupid = get_data.get('wordgroupid', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_wordgroup] collection_del = db_[pongo_wordgroup+'_del'] filter_ = {"_id": ObjectId(wordgroupid)} data = collection.find_one(filter_) data.update({"dateupdated": _datecreated}) collection_del.insert_one(data) res = collection.delete_one(filter_) return JsonResponse({ "code": "0000", "message": "成功", "data": wordgroupid, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 文章模板列表 @csrf_exempt def getContentList(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') categoryid = request.GET.get('categoryid', '') category = request.GET.get('category', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] filter_ = filter_2 = {} if ownerid: filter_ = {"ownerid": ownerid} if owner: filter_ = {"owner": owner} if categoryid: filter_2 = {"categoryid": categoryid} if category: filter_2 = {"category": category} filter_.update(filter_2) pipeline = [ {"$match": filter_}, {"$sort": {"datecreated": -1}}, { "$project": { "_id": 0, "id": {f"$toString": f"$_id"}, "name": 1, "description": 1, "owner": 1, "ownerid": 1, "categoryid": 1, "category": 1, "content": 1, "contenttemplate": 1, "titlelist": 1, "titles": 1, "state": 1, } }, ] res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] if res: data = list(res) return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 文章模板详情 @csrf_exempt def getContent(request): _datecreated = timestamp_to_str() try: if request.method == "GET": industry1 = request.GET.get('industry1', '') industry2 = request.GET.get('industry2', '') orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') viewpointid = request.GET.get('viewpointid', '') viewpointname = request.GET.get('viewpointname', '') ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') templateid = request.GET.get('templateid', '') templatename = request.GET.get('templatename', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] filter_ = {"templateid": templateid} params = { "_id": 0, "id": {f"$toString": f"$_id"}, "name": 1, "description": 1, "owner": 1, "ownerid": 1, "categoryid": 1, "category": 1, "content": 1, "contenttemplate": 1, "wordgroups": 1, "property": 1, "orgid": 1, "orgname": 1, "titlelist": 1, "titles": 1, "industry2": 1, "industryid2": 1, "industry1": 1, "industryid1": 1, "viewpointname": 1, "viewpointid": 1, "templatename": 1, "templatid": 1, "datecreated": 1, "dateupdated": 1, "statistics": 1, "state": 1, } res = collection.find_one(filter_, params, no_cursor_timeout=True) data = {} if res: data = res return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 新增模板 @csrf_exempt def newContent(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) name = get_data.get('name', '') description = get_data.get('description', '') owner = get_data.get('owner', '') ownerid = str(get_data.get('ownerid', '')) categoryid = get_data.get('categoryid', '') category = get_data.get('category', '') content = get_data.get('content', '') wordgroups = get_data.get('wordgroups', []) property = get_data.get('property', {"distinct": 3}) titles = get_data.get('titles', []) contenttemplate = content.replace("【", '').replace("】", '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] data = { "name": name, "titles": titles, "description": description, "owner": owner, "ownerid": ownerid, "categoryid": categoryid, "category": category, "content": content, "contenttemplate": contenttemplate, "wordgroups": wordgroups, "property": property, "datecreated": TIME, "dateupdated": TIME, "sort": 100, "state": 0 } res = collection.insert_one(data) return JsonResponse({ "code": "0000", "message": "成功", "data": res.inserted_id, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 修改模板 @csrf_exempt def updateContent(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) titles = get_data.get('titles', []) content = get_data.get('content', '') wordgroups = get_data.get('wordgroups', []) templateid = get_data.get('templateid', '') templatename = get_data.get('templatename', '') owner = get_data.get('owner', '') ownerid = str(get_data.get('ownerid', '')) orgid = get_data.get('orgid', '') orgname = get_data.get('orgname', '') categoryid = get_data.get('categoryid', '') category = get_data.get('category', '') property = get_data.get('property', {"distinct": 3}) TIME = timestamp_to_str() if type(titles) != type([]): return JsonResponse({"code": "1014", "message": CODE_DICT["1014"]}) with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] params = { "$set": { "titles": titles, "content": content, "wordgroups": wordgroups, "property": property, "dateupdated": TIME, } } filter_ = {"templateid": templateid} res = collection.update_one(filter_, params) return JsonResponse({ "code": "0000", "message": "成功", "data": templateid, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 修改新增模板内容 @csrf_exempt def updateContentTemplateInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) industry1 = get_data.get('industry1', '') industry2 = get_data.get('industry2', '') orgid = get_data.get('orgid', '') orgname = get_data.get('orgname', '') viewpointid = get_data.get('viewpointid', '') viewpointname = get_data.get('viewpointname', '') ownerid = str(get_data.get('ownerid', '')) owner = get_data.get('owner', '') templateid = get_data.get('templateid', '') templatename = get_data.get('templatename', '') contenttemplate = get_data.get('contenttemplate', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] try: params = { "$set": { "contenttemplate": contenttemplate, "dateupdated": TIME, } } filter_ = {"templateid": templateid} res = collection.update_one(filter_, params) except DuplicateKeyError: return JsonResponse({"code": "10003", "message": f"{contenttemplate}-已存在"}) return JsonResponse({ "code": "0000", "message": "成功", "data": templateid, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 删除模板 @csrf_exempt def deleteContent(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) contentid = get_data.get('contentid', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] collection_del = db_[pongo_content + '_del'] filter_ = {"_id": ObjectId(contentid)} data = collection.find_one(filter_) data.update({"dateupdated": _datecreated}) collection_del.insert_one(data) res = collection.delete_one(filter_) return JsonResponse({ "code": "0000", "message": "成功", "data": contentid, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 分类列表 @csrf_exempt def getCategoryList(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_category] filter_ = {} if ownerid: filter_ = {"ownerid": ownerid} if owner: filter_ = {"owner": owner} pipeline = [ {"$match": filter_}, { "$project": { "_id": 0, "id": {f"$toString": f"$_id"}, "name": 1, "owner": 1, "ownerid": 1, } }, ] res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] if res: data = list(res) return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 人物行业列表 @csrf_exempt def getPersonList(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') level = request.GET.get('level', []) orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') # logger.info(f"request.GET: {str(request.GET)}") logger.info(f"level: {str(level)} --- {str(type(level))}") with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_person] filter_ = {} if orgid: filter_ = {"orgid": orgid} if ownerid: filter_.update({"ownerid": ownerid}) if "ORG_ADMIN" in level: filter_.pop("ownerid", '') pipeline = [ {"$match": filter_}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "name": 1, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry": 1, "industryid": 1, "pid": 1, "iid": 1, "orgid": 1, "orgname": 1, } }, ] logger.info(pipeline) res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] temp = {} # if res: # for item in res: # temp.setdefault(item.get('industry1'), []).append({ # "value": item.get('industry2'), # "label": item.get('industry2'), # "industry": item.get('industry'), # "industryid": item.get('industryid'), # }) # for k, v in temp.items(): # t = { # "value": k, # "label": k, # "children": v # } # data.append(t) # res_data = { # "industrylist": data # } t = [] if res: for item in res: # print(item) p_id = item.get('pid') _id = item.get('id') iid = item.get('iid') industryid = item.get('industryid') industry = item.get('industry') if p_id == '0': t.append({"label": industry, "id": _id, "value": industryid, "pid": p_id, "iid": iid}) else: temp.setdefault(p_id, []).append({ "label": industry, "pid": item.get('pid'), "id": _id, "value": industryid, }) for v in t: P_id = v.get('iid') v.update({"children": temp.get(P_id, [])}) data.append(v) return JsonResponse({ "code": "0000", "message": "成功", "data": {"industrylist": data}, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 人物行业详情 @csrf_exempt def getPersonInfo(request): _datecreated = timestamp_to_str() try: if request.method == "GET": personid = request.GET.get('personid', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_person] filter_ = {"_id": ObjectId(personid)} params = { "_id": 0, "name": 1, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industrylist": 1, "orgid": 1, "orgname": 1, } print(filter_) # res = collection.find_one(filter_, no_cursor_timeout=True) res = collection.find_one(filter_, params, no_cursor_timeout=True) data = {} if res: data = res return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 新增人物行业关系 @csrf_exempt def newPersonInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') level = request.GET.get('level', '') orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') industry1 = request.GET.get('industry1', []) industry2 = request.GET.get('industry2', []) TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_person] data = { "level": level, "orgid": orgid, "orgname": orgname, "owner": owner, "ownerid": ownerid, "industry1": industry1, "industry2": industry2, "datecreated": TIME, "dateupdated": TIME, "sort": 100, "state": 0 } res = collection.insert_one(data) return JsonResponse({ "code": "0000", "message": "成功", "data": res.inserted_id, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 修改人物行业关系 @csrf_exempt def updatePersonInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) ownerid = str(get_data.get('ownerid', '')) owner = get_data.get('owner', '') level = get_data.get('level', '') orgid = get_data.get('orgid', '') orgname = get_data.get('orgname', '') children = get_data.get('children', []) TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_industry] collection_p = db_[pongo_person] filter_person = {"orgid": orgid, "ownerid": ownerid} pipeline_person = [ {"$match": filter_person}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry": 1, "industryid": 1, "pid": 1, "iid": 1, "level": 1, "orgid": 1, "orgname": 1, "datecreated": 1, "dateupdated": 1, } }, ] res_person = collection_p.aggregate(pipeline=pipeline_person, allowDiskUse=True) industry_data = {} old_industry_list = [] old_pid_list = [] for item in res_person: _id = item.pop('id') industryid = item.get('industryid') iid = item.get('iid') pid = item.get('pid') old_industry_list.append(iid) old_pid_list.append(pid) industry_data.update(item) collection_p.delete_one({"_id": ObjectId(_id)}) for item in old_industry_list: params = { "$set": { "disabled": False, "dateupdated": TIME, }, } filter_ = {"_id": ObjectId(item)} collection.update_one(filter_, params) for item in children: industry_data_t = {} _id = item item_t = collection.find_one({"_id": ObjectId(_id)}) industry = item_t.get('industry') industryid = item_t.get('industryid') pid = item_t.get('pid') industry_data_t.update(industry_data) industry_data_t.update({ "iid": _id, "ownerid": ownerid, "owner": owner, "level": level, "orgid": orgid, "orgname": orgname, "industry": industry, "industryid": industryid, "pid": pid, "datecreated": TIME, "dateupdated": TIME, }) industry_data_t.pop('_id', '') collection_p.insert_one(industry_data_t) params = { "$set": { "disabled": True, "dateupdated": TIME, }, } filter_ = {"_id": ObjectId(_id)} collection.update_one(filter_, params) return JsonResponse({ "code": "0000", "message": "成功", "data": 0, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 删除人物行业关系 @csrf_exempt def deletePersonInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) orgid = get_data.get('orgid', '') ownerid = str(get_data.get('ownerid', '')) TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection_p = db_[pongo_person] collection_industry = db_[pongo_industry] collection_del = db_[pongo_person+'_del'] filter_person = {"orgid": orgid, "ownerid": ownerid} pipeline_person = [ {"$match": filter_person}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry": 1, "industryid": 1, "pid": 1, "iid": 1, "level": 1, "orgid": 1, "orgname": 1, "datecreated": 1, "dateupdated": 1, } }, ] res_person = collection_p.aggregate(pipeline=pipeline_person, allowDiskUse=True) t = [] if res_person: for item in res_person: # print(item) p_id = item.get('pid') iid = item.get('iid') if p_id != '0': t.append(iid) item.update({"dateupdated": _datecreated}) collection_del.insert_one(item) print(t) # 将行业属性disabled变为false getIndustryList2 for r_id in t: collection_industry.update_one({"_id": ObjectId(r_id)}, {"$set": {"dateupdated": _datecreated, "disabled": False}}) collection_p.delete_many(filter_person) return JsonResponse({ "code": "0000", "message": "成功", "data": "ok", "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 行业列表 @csrf_exempt def getIndustryList(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') level = request.GET.get('level', []) orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_industry] filter_ = {} if orgid: logger.info(str(request.GET)) if "ORG_ADMIN" in level: filter_.update({"orgid": orgid,}) pipeline = [ {"$match": filter_}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry": 1, "industryid": 1, "pid": 1, "disabled": 1, "orgid": 1, "orgname": 1, } }, ] print(pipeline) res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] temp = {} t = [] if res: for item in res: # print(item) p_id = item.get('pid') _id = item.get('id') industryid = item.get('industryid') industry = item.get('industry') if p_id == '0': t.append({"title": industry, "id": _id, "industryid": industryid, "pid": p_id}) else: temp.setdefault(p_id, []).append({ "title": industry, "pid": item.get('pid'), "id": _id, "industryid": industryid, }) for v in t: P_id = v.get('id') v.update({"children": temp.get(P_id, [])}) data.append(v) else: return JsonResponse({"code": "1025", "message": CODE_DICT['1025']}) else: return JsonResponse({"code": "1013", "message": CODE_DICT['1013']}) return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 行业列表--与任务表关联关系 @csrf_exempt def getIndustryList2(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": ownerid = str(request.GET.get('ownerid', '0')) owner = request.GET.get('owner', '') level = request.GET.get('level', []) orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_industry] collection_p = db_[pongo_person] filter_ = {} filter_p = {} if "ORG_ADMIN" in level: filter_.update({"orgid": orgid}) if ownerid: filter_p.update({"ownerid": ownerid, "orgid": orgid}) pipeline = [ {"$match": filter_}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry": 1, "industryid": 1, "pid": 1, "disabled": 1, "orgid": 1, "orgname": 1, } }, ] pipeline_p = [ {"$match": filter_p}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry": 1, "industryid": 1, "disabled": 1, "orgid": 1, "orgname": 1, "pid": 1, "iid": 1, } }, ] print(pipeline) res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) res2 = collection_p.aggregate(pipeline=pipeline_p, allowDiskUse=True) if res2: temp_2 = {} t_2 = [] for item in res2: # print(item) p_id = item.get('pid') i_id = item.get('iid') _id = item.get('id') industryid = item.get('industryid') industry = item.get('industry') if p_id == '0': t_2.append({"title": industry, "id": _id, "industryid": industryid, "pid": p_id}) else: temp_2.setdefault(i_id, []).append({ "title": industry, "pid": item.get('pid'), "id": _id, "industryid": industryid, }) if res: data = [] temp = {} t = [] for item in res: # print(item) p_id = item.get('pid') _id = item.get('id') industryid = item.get('industryid') industry = item.get('industry') disabled = item.get('disabled') if p_id == '0': t.append({"title": industry, "id": _id, "industryid": industryid, "pid": p_id, "expand": True, "disabled": True,}) else: temp.setdefault(p_id, []).append({ "title": industry, "pid": item.get('pid'), "id": _id, "industryid": industryid, "disabled": disabled, }) print("pipeline_person", temp_2) for v in t: P_id = v.get('id') children = temp.get(P_id, []) children_new = [] for i in children: if i.get('id') in temp_2: i.update({"checked": True}) i.pop("disabled") else: i.update({"checked": False}) children_new.append(i) v.update({"children": children_new}) data.append(v) # ##################### # p_industry = {} # for item in res2: # p_industry.setdefault(item.get('industry1'), []).append(item.get('industry2')) # data = [] # temp = {} # d_temp = {} # if res: # for item in res: # temp.setdefault(item.get('industry1'), []).append({"title": item.get('industry2'), "disabled": item.get('disabled'), "id": item.get("id")}) # d_temp.setdefault(item.get('industry1'), []).append(item.get('industry2')) # # "checked ": True # print(temp) # for k, v in temp.items(): # vv_new = [] # for vv in v: # if vv.get('title', '') in p_industry.get(k, []): # vv.update({"checked": True, "title": vv.get('title', ''),}) # vv.pop("disabled") # else: # vv.update({"checked": False, "title": vv.get('title', ''), "disabled": vv.get('disabled', '')}) # vv_new.append(vv) # t = { # "title": k, # "expand": True, # "disabled": True, # "children": vv_new # } # data.append(t) # ##################### return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 行业详情 @csrf_exempt def getIndustryInfo(request): _datecreated = timestamp_to_str() try: if request.method == "GET": industryid = request.GET.get('industryid', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_industry] filter_ = {"_id": ObjectId(industryid)} params = { "_id": 0, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "orgid": 1, "orgname": 1, } print(filter_) # res = collection.find_one(filter_, no_cursor_timeout=True) res = collection.find_one(filter_, params, no_cursor_timeout=True) data = {} if res: data = res return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 新增行业 @csrf_exempt def newIndustryInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) ownerid = str(get_data.get('ownerid', '')) owner = get_data.get('owner', '') level = get_data.get('level', []) orgid = get_data.get('orgid', '') orgname = get_data.get('orgname', '') industry = get_data.get('title', '') pid = get_data.get('pid', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_industry] collection_industry_ids = db_[pongo_industry_ids] data = {} ids_dict = collection_industry_ids.find_one({"id": 1}) industry1id = int(ids_dict.get('industry1id')) industry2id = int(ids_dict.get('industry2id')) if 'ORG_ADMIN' in level: if pid == '0': # 新增一级 data_indsert = { "level": level, "orgid": orgid, "orgname": orgname, "owner": owner, "ownerid": ownerid, "industry": industry, "industryid": str(industry1id), "disabled": False, "datecreated": TIME, "dateupdated": TIME, "sort": 100, "state": 0, "pid": "0" } res = collection.insert_one(data_indsert) pipeline = [ {"$match": {"_id": ObjectId(res.inserted_id)}}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry": 1, "level": 1, "orgid": 1, "orgname": 1, "datecreated": 1, "dateupdated": 1, "pid": 1, } }, ] data = collection.aggregate(pipeline=pipeline, allowDiskUse=True) collection_industry_ids.update_one({"id": 1}, {"$set": {"industry1id": str(industry1id + 1), "dateupdated": TIME}}) else: ids_dict = collection.find_one({"_id": ObjectId(pid)}) # 新增二级 data_indsert = { "level": level, "orgid": orgid, "orgname": orgname, "owner": owner, "ownerid": ownerid, "industry": industry, "industryid": ids_dict.get("industryid") + str(industry2id), "disabled": False, "datecreated": TIME, "dateupdated": TIME, "sort": 100, "state": 0, "pid": pid, } res = collection.insert_one(data_indsert) pipeline = [ {"$match": {"_id": res.inserted_id}}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry": 1, "level": 1, "orgid": 1, "orgname": 1, "industryid": 1, "datecreated": 1, "dateupdated": 1, "pid": 1, } }, ] data = collection.aggregate(pipeline=pipeline, allowDiskUse=True) collection_industry_ids.update_one({"id": 1}, { "$set": {"industry2id": str(industry2id + 1), "dateupdated": TIME}}) else: return JsonResponse({"code": "1010", "message": CODE_DICT['1010']}) return JsonResponse({ "code": "0000", "message": "成功", "data": list(data), "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 修改行业 @csrf_exempt def updateIndustryInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) ownerid = str(get_data.get('ownerid', '')) owner = get_data.get('owner', '') level = get_data.get('level', '') orgid = get_data.get('orgid', '') orgname = get_data.get('orgname', '') indusrty = get_data.get('title', '') indusrtyid = get_data.get('indusrtyid', '') industry_id = get_data.get('id') TIME = timestamp_to_str() logger.info(str(get_data)) with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_industry] collection_p = db_[pongo_person] filter_industry = {"_id": ObjectId(industry_id)} logger.info(f"filter_industry:" + str(filter_industry)) params_i = { "$set": { "industry": indusrty, "dateupdated": TIME, }, } r_i = collection.update_one(filter_industry, params_i) # ret.matched_count, ret.modified_count # 前者是匹配到文档的数目,而后者是修改了文档的个数。 logger.info(f"匹配:" + str(r_i.matched_count)) logger.info(f"修改:" + str(r_i.modified_count)) params_p = { "$set": { "industry": indusrty, "dateupdated": TIME, }, } filter_p = {"iid": industry_id} r_p = collection_p.update_many(filter_p, params_p) logger.info(f"匹配:" + str(r_p.matched_count)) logger.info(f"修改:" + str(r_p.modified_count)) return JsonResponse({ "code": "0000", "message": "成功", "data": 0, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 删除行业 @csrf_exempt def deleteIndustryInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) industryid = get_data.get('id', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_industry] collection_del = db_[pongo_industry+'_del'] filter_ = {"_id": ObjectId(industryid)} data = collection.find_one(filter_) data.update({"dateupdated": _datecreated}) pid = data.get('pid') disabled = data.get('disabled') if pid == '0': filter_ = {"pid": industryid} pipeline = [ {"$match": filter_}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry": 1, "level": 1, "orgid": 1, "orgname": 1, "industryid": 1, "datecreated": 1, "dateupdated": 1, "pid": 1, "disabled": 1, } }, ] data = collection.aggregate(pipeline=pipeline, allowDiskUse=True) flag = 0 data_t = [] for item in data: disabled = item.get('disabled', True) if disabled == True: flag = 1 break data_t.append(item) if flag: code = "1011" message = "此一级行业已经分配二级行业,不能删除" else: for item in data_t: item.update({"dateupdated": _datecreated}) collection_del.insert_one(item) collection.delete_many(filter_) filter_ = {"_id": ObjectId(industryid)} collection.delete_one(filter_) code = "0000" message = "删除成功" else: if disabled == False: data.update({"dateupdated": _datecreated}) collection_del.insert_one(data) collection.delete_one(filter_) code = "0000" message = "删除成功" else: code = "1012" message = "行业已分配角色,不能删除" return JsonResponse({ "code": code, "message": CODE_DICT[code], "data": "ok", "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # ################################## # 关键词 # ################################## # 发版词上传 @csrf_exempt def baseKeywordUpload(request): _datecreated = timestamp_to_str() if request.method == 'POST': try: with MongoDBClient() as mg_client, LogMysqlApeT(ape_db) as m_client: db_ = mg_client[db] # 获取collection collection = db_[pongo_keyword] path_info = request.path_info jgname = path_info.split('/')[-1] file_obj_list = request.FILES.getlist('file') orgid = request.POST.get('orgid', '') orgname = request.POST.get('orgname', '') sql = f"select id,wordgroupname from {simian_wordgroup_table} where orgid='{orgid}' and status=0 " res_w = m_client.search_db(sql) wordgroupname_dict = {} for item_w in res_w: wordgroupname_dict.update({item_w.get("wordgroupname"): item_w.get("id")}) file_path_list = [] filePath = [] flag = 0 for file_obj in file_obj_list: file_path = r'uploadFiles' + os.sep + file_obj.name file_path_list.append(file_path) filePath.append(file_obj.name) with open(file_path, 'wb') as f: for i in file_obj.chunks(): f.write(i) logger.info('文件路径:%s' % str(file_path)) res = read_excel(file_path) error_list = [["提示信息", "关键词名称", "词分组名称", "一级行业", "二级行业"]] collection_industry = db_[pongo_industry] industry_dict = {} res_all = collection_industry.find({}) for item_i in res_all: pid = item_i.get('pid') industry = item_i.get('industry') industryid = item_i.get('industryid') if pid == '0': industry = industry + '_1' else: industry = industry + '_2' industry_dict.update({industry: industryid}) pass logger.info(str(industry_dict)) count = 0 for item in res: # logger.info(str(item)) # print(len(item), item) TIME = timestamp_to_str(time.time() + count) if item[1]: keyword = item[3] industry1 = item[1] industry2 = item[2] area = item[4] baiduindex = item[5] wordgroup = item[6] industryid2 = industry_dict.get(industry2 + '_2', '') industryid1 = industry_dict.get(industry1 + '_1', '') if not industryid1 or not industryid2: k_t = ["该关键词对应行业不存在", keyword, wordgroup, industry1, industry2] error_list.append(k_t) count += 1 continue if wordgroup not in wordgroupname_dict: k_t = ["该分组不存在", keyword, wordgroup, industry1, industry2] error_list.append(k_t) count += 1 continue data = { "keyword": keyword, "industry1": industry1, "industryid1": industryid1, "industry2": industry2, "industryid2": industryid2, "baiduindex": baiduindex, "area": area, "orgid": orgid, "orgname": orgname, "datecreated": TIME, "dateupdated": TIME, "sort": 100, "state": 0, "publishlist": [], "wordgroup": wordgroup, "wordgroupid": wordgroupname_dict.get(wordgroup, ''), } try: collection.insert_one(data) except DuplicateKeyError: k_t = ["该关键词已存在", keyword, wordgroup, industry1, industry2] error_list.append(k_t) count += 1 # 上传成功之后删除文件 for path in file_path_list: try: os.remove(path) except Exception as e: pass if len(error_list) != 1: code = "1026" return JsonResponse( {"code": code, "data": error_list, "message": CODE_DICT[code], "datecreated": _datecreated, }) else: code = "0000" error_list = [] return JsonResponse( {"code": code, "data": error_list, "message": CODE_DICT[code], "datecreated": _datecreated, }) except Exception as e: return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated,}) return JsonResponse({"code": "00", "message": "no this method", "datecreated": _datecreated,}) # 发布任务对应词和站点关系上传上传 @csrf_exempt def publishKeywordUpload(request): _datecreated = timestamp_to_str() if request.method == 'POST': try: logger.info('# 开始上传文件 #') with MongoDBClient() as mg_client, LogMysqlApeT(ape_db) as m_client: db_ = mg_client[db] # 获取collection collection = db_[pongo_keyword] path_info = request.path_info jgname = path_info.split('/')[-1] file_obj_list = request.FILES.getlist('file') aid = request.POST.get('aid', '') aname = request.POST.get('aname', '') orgid = request.POST.get('orgid', '') orgname = request.POST.get('orgname', '') # sql = f"select id,wordgroupname from {simian_wordgroup_table} where orgid='{orgid}' and status=0 " # res_w = m_client.search_db(sql) # wordgroupname_dict = {} # for item_w in res_w: # wordgroupname_dict.update({item_w.get("wordgroupname"): item_w.get("id")}) file_path_list = [] filePath = [] for file_obj in file_obj_list: file_path = r'uploadFiles' + os.sep + file_obj.name file_path_list.append(file_path) filePath.append(file_obj.name) with open(file_path, 'wb') as f: for i in file_obj.chunks(): f.write(i) logger.info('文件路径:%s' % str(file_path)) res = read_excel(file_path) TIME = timestamp_to_str(time.time()) error_list = [["提示信息", "词组名称", "关键词名称"]] for item in res: if item[0]: _id = item[0] keyword = item[1] industry1 = item[2] industry2 = item[3] area = item[4] baiduindex = item[5] wordgroup = item[6] # if wordgroup not in wordgroupname_dict: # k_t = ["该分组不存在", wordgroup, keyword] # error_list.append(k_t) # continue labelname1 = item[7] labelname2 = item[8] labelname3 = item[9] # 0 待编辑 # 1 已编辑 # 2 已发布 params = { "$push": {"publishlist": { "id": get_uuid(), "aid": str(aid), "aname": aname, "labelname1": labelname1, "labelname2": labelname2, "labelname3": labelname3, "title": "", # 看不见 计划中 "status": 10, "onlinedate": "", }}, "$set": { "dateupdated": TIME, # "wordgroup": wordgroup, # "wordgroupid": wordgroupname_dict.get(wordgroup, ''), }, } filter_ = {"_id": ObjectId(_id)} res = collection.update_one(filter_, params) yes = res.modified_count if not yes: k_t = ["该词对应id不存在", wordgroup, keyword] error_list.append(k_t) continue # 上传成功之后删除文件 for path in file_path_list: try: os.remove(path) except Exception as e: pass # 增删改查 if len(error_list) != 1: code = "1026" return JsonResponse({"code": code, "data": error_list, "message": CODE_DICT[code], "datecreated": _datecreated,}) else: code = "0000" error_list = [] return JsonResponse({"code": code, "data": error_list, "message": CODE_DICT[code], "datecreated": _datecreated,}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated,}) return JsonResponse({"code": "00", "message": "no this method", "datecreated": _datecreated,}) # 关键词列表 --- 基础词任务 @csrf_exempt def getKeywordList(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') keyword = request.GET.get('keyword', '') area = request.GET.get('area', '') wordgroup = request.GET.get('wordgroup', '') wordgroupid = request.GET.get('wordgroupid', '') orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') industry1 = request.GET.get('industry1', '') industryid1 = request.GET.get('industryid1', '') industry2 = request.GET.get('industry2', '') industryid2 = request.GET.get('industryid2', '') page = int(request.GET.get('page', 1)) size = int(request.GET.get('size', 10)) with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_keyword] filter_ = {} if orgid: filter_.update({"orgid": orgid}) if industryid1: filter_.update({"industryid1": industryid1}) if industryid2: filter_.update({"industryid2": industryid2}) if area: filter_.update({"area": area}) if wordgroup: filter_.update({"wordgroup": wordgroup}) if keyword: filter_.update({"keyword": {"$regex": keyword}}) pipeline = [ {"$match": filter_}, {"$sort": {"datecreated": -1}}, {"$limit": page * size}, {"$skip": (page - 1) * size}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "wordgroup": 1, "wordgroupid": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "keyword": 1, "area": 1, "orgid": 1, "orgname": 1, "baiduindex": 1, } }, ] print(pipeline) total = collection.count_documents(filter=filter_) res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] if res: data = list(res) else: code = "1016" return JsonResponse({"code": code, "message": CODE_DICT[code]}) return JsonResponse({ "code": "0000", "message": "成功", "total": total, "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 关键词列表 --- 基础词任务 ---- 下载 @csrf_exempt def getKeywordListDownload(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') keyword = request.GET.get('keyword', '') area = request.GET.get('area', '') wordgroup = request.GET.get('wordgroup', '') wordgroupid = request.GET.get('wordgroupid', '') orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') industry1 = request.GET.get('industry1', '') industryid1 = request.GET.get('industryid1', '') industry2 = request.GET.get('industry2', '') industryid2 = request.GET.get('industryid2', '') page = int(request.GET.get('page', 1)) size = int(request.GET.get('size', 10)) with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_keyword] filter_ = {} if orgid: filter_.update({"orgid": orgid}) if industryid1: filter_.update({"industryid1": industryid1}) if industryid2: filter_.update({"industryid2": industryid2}) if area: filter_.update({"area": area}) if wordgroup: filter_.update({"wordgroup": wordgroup}) if keyword: filter_.update({"keyword": {"$regex": keyword}}) pipeline = [ {"$match": filter_}, {"$sort": {"datecreated": -1}}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "wordgroup": 1, "wordgroupid": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "keyword": 1, "area": 1, "orgid": 1, "orgname": 1, "baiduindex": 1, } }, ] print(pipeline) total = collection.count_documents(filter=filter_) res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] if res: data = list(res) else: code = "1016" return JsonResponse({"code": code, "message": CODE_DICT[code]}) return JsonResponse({ "code": "0000", "message": "成功", "total": total, "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 关键词列表 --- 发布任务 词 @csrf_exempt def getTaskKeywordList(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') keyword = request.GET.get('keyword', '') area = request.GET.get('area', '') aid = int(request.GET.get('aid', 0)) wordgroup = request.GET.get('wordgroup', '') wordgroupid = request.GET.get('wordgroupid', '') orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') industry1 = request.GET.get('industry1', '') industryid1 = request.GET.get('industryid1', '') industry2 = request.GET.get('industry2', '') industryid2 = request.GET.get('industryid2', '') page = int(request.GET.get('page', 1)) size = int(request.GET.get('size', 10)) with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_keyword] filter_ = {} if orgid: filter_.update({"orgid": orgid}) if industryid1: filter_.update({"industryid1": industryid1}) if industryid2: filter_.update({"industryid2": industryid2}) if area: filter_.update({"area": area}) if wordgroup: filter_.update({"wordgroup": wordgroup}) if keyword: filter_.update({"keyword": {"$regex": keyword}}) if aid: filter_.update({"publishlist": {"$elemMatch": {"aid": str(aid)}}}) pipeline = [ {"$match": filter_}, {"$sort": {"keyword": 1}}, {"$limit": page * size}, {"$skip": (page - 1) * size}, {"$unwind": {"path": "$publishlist", "preserveNullAndEmptyArrays": True}}, # { # "$group": { # "_id": "$publishlist.aid", # }, # }, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "wordgroup": 1, "wordgroupid": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "keyword": 1, "area": 1, "orgid": 1, "orgname": 1, "baiduindex": 1, "paid": "$publishlist.id", "aid": "$publishlist.aid", "aname": "$publishlist.aname", "labelname1": "$publishlist.labelname1", "labelname2": "$publishlist.labelname2", "labelname3": "$publishlist.labelname3", } }, ] if aid: pipeline.insert(-1, {"$match": {"publishlist.aid": str(aid)}},) logger.info(pipeline) total = collection.count_documents(filter=filter_) res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] if res: data = list(res) else: code = "1016" return JsonResponse({"code": code, "message": CODE_DICT[code]}) return JsonResponse({ "code": "0000", "message": "成功", "total": total, "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 关键词详情 @csrf_exempt def getKeywordInfo(request): _datecreated = timestamp_to_str() try: if request.method == "GET": keywordid = request.GET.get('keywordid', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_keyword] filter_ = {"_id": ObjectId(keywordid)} params = { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "wordgroupid": 1, "wordgroup": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "keyword": 1, "area": 1, "orgid": 1, "orgname": 1, } print(filter_) # res = collection.find_one(filter_, no_cursor_timeout=True) res = collection.find_one(filter_, params, no_cursor_timeout=True) data = {} if res: data = res return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 关键词新增 @csrf_exempt def newKeywordInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) keyword = get_data.get('keyword', '') wordgroup = get_data.get('wordgroup', '') wordgroupid = get_data.get('wordgroupid', '') area = get_data.get('area', '') ownerid = str(get_data.get('ownerid', '')) owner = get_data.get('owner', '') # level = get_data.get('level', []) orgid = get_data.get('orgid', '') orgname = get_data.get('orgname', '') industry1 = get_data.get('industry1', '') industryid1 = get_data.get('industryid1', '') industry2 = get_data.get('industry2', '') industryid2 = get_data.get('industryid2', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_keyword] data = { "orgid": orgid, "keyword": keyword, "orgname": orgname, "wordgroup": wordgroup, "wordgroupid": wordgroupid, "area": area, "owner": owner, "ownerid": ownerid, "industry1": industry1, "industryid1": industryid1, "industry2": industry2, "industryid2": industryid2, "datecreated": TIME, "dateupdated": TIME, "publishlist": [], "publishsum": 5, "statevalue": "待编辑", "sort": 100, "state": 0 } try: res = collection.insert_one(data) except DuplicateKeyError: code = "1018" return JsonResponse({"code": code, "message": f"({keyword})-{CODE_DICT[code]}"}) return JsonResponse({ "code": "0000", "message": "成功", "data": res.inserted_id, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 关键词编辑 @csrf_exempt def updateKeywordInfo(request): logger.info("updateKeywordInfo") _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) keywordid = get_data.get('keywordid', '') keywordlist = get_data.get('keywordlist', []) # keywordlist = json.loads(keywordlist) wordgroup = get_data.get('wordgroup', '') wordgroupid = get_data.get('wordgroupid', '') keyword = get_data.get('keyword', '') area = get_data.get('area', '') logger.info(f'{str(get_data)}') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_keyword] xgz = {} if keyword: xgz.update({"keyword": keyword}) if keyword: xgz.update({"area": area}) if wordgroupid: xgz.update({"wordgroup": wordgroup, "wordgroupid": wordgroupid}) xgz.update({"dateupdated": TIME}) params = { "$set": xgz, } code = "0000" if keywordid: try: filter_ = {"_id": ObjectId(keywordid)} res = collection.update_one(filter_, params) except DuplicateKeyError: code = "1018" return JsonResponse({"code": code, "message": f"({keyword})-{CODE_DICT[code]}"}) count_all = len(keywordlist) count = 0 if keywordlist: flag = 0 error_list = [] for item in keywordlist: try: keywordid_t = item.get('id') keyword_t = item.get('keyword') filter_t = {"_id": ObjectId(keywordid_t)} res = collection.update_one(filter_t, params) count += int(res.modified_count) except DuplicateKeyError: flag = 1 error_list.append(keyword_t) code = "1018" if flag: return JsonResponse({"code": code, "message": f"({','.join(error_list)})-{CODE_DICT[code]}"}) return JsonResponse({ "code": code, "message": CODE_DICT[code], "data": f"一共修改{count_all}个,成功{count}个!", "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 关键词删除 @csrf_exempt def deleteKeywordInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) keywordid = get_data.get('keywordid', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_keyword] collection_del = db_[pongo_keyword+'_del'] filter_ = {"_id": ObjectId(keywordid)} data = collection.find_one(filter_) data.update({"dateupdated": _datecreated}) collection_del.insert_one(data) res = collection.delete_one(filter_) return JsonResponse({ "code": "0000", "message": "成功", "data": keywordid, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 词分类列表- 有分页 @csrf_exempt def getWordCategoryList(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": page = int(request.GET.get('page', 1)) size = int(request.GET.get('size', 10)) ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') wordgroup = request.GET.get('wordgroup', '') wordgroupid = request.GET.get('wordgroupid', '') orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') industry1 = request.GET.get('industry1', '') industryid1 = request.GET.get('industryid1', '') industry2 = request.GET.get('industry2', '') industryid2 = request.GET.get('industryid2', '') description = request.GET.get("description", '') # 描述 data = [] if orgid: sql = f" select * from {simian_wordgroup_table} where (status=0) " total_sql = f" select count(*) as n from {simian_wordgroup_table} where (status=0) " sql += f" and orgid='{orgid}' " total_sql += f" and orgid='{orgid}' " # if ownerid: # sql += f" and ownerid='{ownerid}' " # total_sql += f" and ownerid='{ownerid}' " if industryid1: sql += f" and industryid1='{industryid1}' " total_sql += f" and industryid1='{industryid1}' " if industryid2: sql += f" and industryid2='{industryid2}' " total_sql += f" and industryid2='{industryid2}' " if wordgroup: sql += f" and wordgroupname like '%%{wordgroup}%%' " total_sql += f" and wordgroupname like '%%{wordgroup}%%' " sql += f" order by createtime DESC limit {(page-1)* size}, {size*page} " with LogMysqlApeT(ape_db) as m_client: data = m_client.search_db(sql) total_ = m_client.search_db(total_sql) total = 0 if total_: total = total_[0].get('n') return JsonResponse({ "code": "0000", "message": "成功", "total": total, "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) # else: # code = "1017" # return JsonResponse({"code": code, "message": CODE_DICT[code]}) else: code = "1016" return JsonResponse({"code": code, "message": CODE_DICT[code]}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 词分类列表- 没有分页 @csrf_exempt def getWordCategoryListAll(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": page = int(request.GET.get('page', 1)) size = int(request.GET.get('size', 10)) ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') wordgroup = request.GET.get('wordgroup', '') wordgroupid = request.GET.get('wordgroupid', '') orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') industry1 = request.GET.get('industry1', '') industryid1 = request.GET.get('industryid1', '') industry2 = request.GET.get('industry2', '') industryid2 = request.GET.get('industryid2', '') description = request.GET.get("description", '') # 描述 data = [] if orgid: sql = f" select wordgroupname as wordgroup, id from {simian_wordgroup_table} where (status=0) " total_sql = f" select count(*) as n from {simian_wordgroup_table} where (status=0) " sql += f" and orgid='{orgid}' " total_sql += f" and orgid='{orgid}' " # if ownerid: # sql += f" and ownerid='{ownerid}' " # total_sql += f" and ownerid='{ownerid}' " if industryid1: sql += f" and industryid1='{industryid1}' " total_sql += f" and industry1id='{industryid1}' " if industryid2: sql += f" and industryid2='{industryid2}' " total_sql += f" and industryid2='{industryid2}' " sql += f" order by createtime DESC " # sql += f" limit {(page-1)* size}, {size*page} " with LogMysqlApeT(ape_db) as m_client: data = m_client.search_db(sql) # total_ = m_client.search_db(total_sql) # total = 0 # if total_: # total = total_[0].get('n') return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) else: code = "1019" return JsonResponse({"code": code, "message": CODE_DICT[code]}) else: code = "1016" return JsonResponse({"code": code, "message": CODE_DICT[code]}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 词分类详情 @csrf_exempt def getWordCategoryInfo(request): _datecreated = timestamp_to_str() try: if request.method == "GET": wordgroupid = request.GET.get('wordgroupid', '') with LogMysqlApeT(ape_db) as m_client: sql = f" select * from {simian_wordgroup_table} where id='{wordgroupid}' " res = m_client.search_db(sql) data = {} if res: data = res[0] return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 词分类添加 @csrf_exempt def newWordCategoryInfo(request): if request.method == 'POST': get_data = {} try: with LogMysqlApeT(ape_db) as m_client: get_data = json.loads(request.body) industry1 = get_data.get("industry1", '') # 行业 industryid1 = get_data.get("industryid1", '') # 行业 industry2 = get_data.get("industry2", '') # 行业 industryid2 = get_data.get("industryid2", '') # 行业 wgname = get_data.get("wordgroup", '') # 词分类名称 owner = get_data.get("owner", '') # 所属人 ownerid = get_data.get("ownerid", '') # 所属人id orgname = get_data.get("orgname", '') # 机构名称 orgid = get_data.get("orgid", '') # 机构id description = get_data.get("description", '') # 描述 # 查询词是否存在 condition = f""" select count(id) as count from {simian_wordgroup_table} where wordgroupname = '{wgname}' and orgid='{orgid}' and industryid2='{industryid2}' and status=0 """ logger.info(condition) wgname_count = m_client.search_db(condition) if wgname_count[0]["count"] > 0: return JsonResponse({"code": "1015", "data": wgname_count[0]["count"], "message": CODE_DICT['1015']}) else: TIME = timestamp_to_str(time.time()) data = { "id": f"wg-{ownerid}-" + get_uuid(), "industry": industry1, "industryid1": industryid1, "industryid2": industryid2, "industry2": industry2, "wordgroupname": wgname, "owner": owner, "ownerid": ownerid, "orgname": orgname, "orgid": orgid, "description": description, "createtime": TIME, "updatetime": TIME, "status": 0 } result = m_client.updata_db(data, simian_wordgroup_table) return JsonResponse({"code": "0000", "data": result, "message": 'success'}) except Exception as e: return JsonResponse({"code": "1002", "msg": str(e)}) return JsonResponse({"code": "1001", "msg": "no this method"}) # 词分类编辑 @csrf_exempt def updateWordCategoryInfo(request): if request.method == 'POST': get_data = {} try: with LogMysqlApeT(ape_db) as m_client: get_data = json.loads(request.body) industry1 = get_data.get("industry1", '') # 行业 industryid1 = get_data.get("industryid1", '') # 行业 industry2 = get_data.get("industry2", '') # 行业 industryid2 = get_data.get("industryid2", '') # 行业 wordgroupid = get_data.get("wordgroupid", '') # 词分类名称 wgname = get_data.get("wordgroup", '') # 词分类名称 owner = get_data.get("owner", '') # 所属人 ownerid = get_data.get("ownerid", '') # 所属人id orgname = get_data.get("orgname", '') # 机构名称 orgid = get_data.get("orgid", '') # 机构id description = get_data.get("description", '') # 描述 TIME = timestamp_to_str(time.time()) # 查询词是否存在 condition = f""" select count(id) as count from {simian_wordgroup_table} where wordgroupname = '{wgname}' and orgid='{orgid}' and industryid2='{industryid2}' and status=0 """ logger.info(condition) wgname_count = m_client.search_db(condition) if wgname_count[0]["count"] > 0: return JsonResponse( {"code": "1015", "data": wgname_count[0]["count"], "message": CODE_DICT['1015']}) data = { "id": wordgroupid, "wordgroupname": wgname, "description": "修改了词分组", "createtime": TIME, "updatetime": TIME } result = m_client.updata_db(data, simian_wordgroup_table) return JsonResponse({"code": "0000", "data": result, "message": 'success'}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 词分类删除 @csrf_exempt def deleteWordCategoryInfo(request): if request.method == 'POST': get_data = {} try: with LogMysqlApeT(ape_db) as m_client: get_data = json.loads(request.body) wordgroupid = get_data.get("wordgroupid", '') # 词分类id TIME = timestamp_to_str(time.time()) res = m_client.search_db( f"""select wordgroupname from {simian_wordgroup_table} where id= '{wordgroupid}' """) v = res[0].get('wordgroupname', '') if res else wordgroupid data = { "id": wordgroupid, "wordgroupname": v+'___', "status": 1, "updatetime": TIME } result = m_client.updata_db(data, simian_wordgroup_table) return JsonResponse({"code": "0000", "data": result, "message": 'success'}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 观点列表 @csrf_exempt def getViewPointList(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": viewpointid = request.GET.get('viewpointid', '') viewpointname = request.GET.get('viewpointname', '') description = request.GET.get('description', '') ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') orgid = request.GET.get('orgid', '') page = int(request.GET.get('page', 1)) size = int(request.GET.get('size', 10)) orgname = request.GET.get('orgname', '') industry1 = request.GET.get('industry1', '') industry2 = request.GET.get('industry2', '') industryid1 = request.GET.get('industryid1', '') industryid2 = request.GET.get('industryid2', '') data = [] sql = f" select * from {simian_viewpoint_table} where (status=0) " total_sql = f" select count(*) as n from {simian_viewpoint_table} where (status=0) " if orgid: sql += f" and orgid='{orgid}' " total_sql += f" and orgid='{orgid}' " # if ownerid: # sql += f" and ownerid='{ownerid}' " # total_sql += f" and ownerid='{ownerid}' " if industryid1: sql += f" and industryid1='{industryid1}' " total_sql += f" and industryid1='{industryid1}' " if industryid2: sql += f" and industryid2='{industryid2}' " total_sql += f" and industryid2='{industryid2}' " if viewpointname: sql += f" and viewpoint like '%%{viewpointname}%%' " total_sql += f" and viewpoint like '%%{viewpointname}%%' " # if description: # sql += f" and description like '%%{description}%%' " # total_sql += f" and description like '%%{description}%%' " sql += f" order by createtime DESC limit {(page - 1) * size}, {size * page} " with LogMysqlApeT(ape_db) as m_client: data = m_client.search_db(sql) total_ = m_client.search_db(total_sql) total = 0 if total_: total = total_[0].get('n') return JsonResponse({ "code": "0000", "message": "成功", "total": total, "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) else: code = "1021" return JsonResponse({"code": code, "message": CODE_DICT[code]}) else: code = "1016" return JsonResponse({"code": code, "message": CODE_DICT[code]}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 观点详情 @csrf_exempt def getViewPointInfo(request): _datecreated = timestamp_to_str() try: if request.method == "GET": viewpointid = request.GET.get('viewpointid', '') with LogMysqlApeT(ape_db) as m_client: sql = f" select * from {simian_viewpoint_table} where id='{viewpointid}' " res = m_client.search_db(sql) data = {} if res: data = res[0] return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 观点添加 @csrf_exempt def newViewPointInfo(request): if request.method == 'POST': get_data = {} try: with LogMysqlApeT(ape_db) as m_client: get_data = json.loads(request.body) industry = get_data.get("industry1", '') # 行业 industry2 = get_data.get("industry2", '') # 行业 viewpointname = get_data.get("viewpointname", '') # 观点名称 owner = get_data.get("owner", '') # 所属人 ownerid = get_data.get("ownerid", '') # 所属人id orgname = get_data.get("orgname", '') # 机构名称 orgid = get_data.get("orgid", '') # 机构id description = get_data.get("description", '') # 写作方向 industryid1 = get_data.get('industryid1', '') industryid2 = get_data.get('industryid2', '') # 查询词是否存在 condition = f""" select count(id) as count from {simian_viewpoint_table} where viewpoint = '{viewpointname}' and orgid='{orgid}' and industryid2='{industryid2}' """ logger.info(condition) viewpointgnamec = m_client.search_db(condition) if viewpointgnamec[0]["count"] > 0: return JsonResponse({"code": "0000", "data": viewpointgnamec[0]["count"], "message": '该观点名称已存在'}) TIME = timestamp_to_str(time.time()) data = { "id": f"vp-{ownerid}-" + get_uuid(), "industry": industry, "industry2": industry2, "industryid1": industryid1, "industryid2": industryid2, "viewpoint": viewpointname, "description": description, "owner": owner, "ownerid": ownerid, "orgname": orgname, "orgid": orgid, "createtime": TIME, "updatetime": TIME, "status": 0 } result = m_client.updata_db(data, simian_viewpoint_table) return JsonResponse({"code": "0000", "data": result, "message": 'success'}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 观点编辑 @csrf_exempt def updateViewPointInfo(request): if request.method == 'POST': get_data = {} try: with LogMysqlApeT(ape_db) as m_client: get_data = json.loads(request.body) viewpointid = get_data.get("viewpointid", '') # 观点id viewpointname = get_data.get("viewpointname", '') # 观点id description = get_data.get("description", '') # 观点id TIME = timestamp_to_str(time.time()) data = { "id": viewpointid, "viewpoint": viewpointname, "description": description, "updatetime": TIME } result = m_client.updata_db(data, simian_viewpoint_table) return JsonResponse({"code": "0000", "data": result, "message": 'success'}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 观点删除 @csrf_exempt def deleteViewPointInfo(request): if request.method == 'POST': get_data = {} try: with LogMysqlApeT(ape_db) as m_client: get_data = json.loads(request.body) viewpointid = get_data.get("viewpointid", '') # 观点id res = m_client.search_db(f"""select viewpoint from {simian_viewpoint_table} where id= '{viewpointid}' """) v = res[0].get('viewpoint', '') if res else viewpointid TIME = timestamp_to_str(time.time()) data = { "id": viewpointid, "viewpoint": v + '__', "status": 1, "updatetime": TIME } result = m_client.updata_db(data, simian_viewpoint_table) return JsonResponse({"code": "0000", "data": result, "message": 'success'}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 观点模板列表 @csrf_exempt def getTemplateList(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') viewpointid = request.GET.get('viewpointid', '') viewpointname = request.GET.get('viewpointname', '') ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') industryid1 = request.GET.get('industryid1', '') industryid2 = request.GET.get('industryid2', '') state = int(request.GET.get('state', 10000)) with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] filter_ = filter_2 = {} if orgid: filter_.update({"orgid": orgid}) if industryid1: filter_.update({"industryid1": industryid1}) if industryid2: filter_.update({"industryid2": industryid2}) if viewpointid: filter_.update({"viewpointid": viewpointid}) if state != 10000: filter_.update({"state": state}) pipeline = [ {"$match": filter_}, {"$sort": {"datecreated": -1}}, { "$project": { "_id": 0, "id": {f"$toString": f"$_id"}, "orgid": 1, "orgname": 1, "owner": 1, "ownerid": 1, "viewpointid": 1, "viewpointname": 1, "industry1": 1, "industry2": 1, "industryid1": 1, "industryid2": 1, "templateid": 1, "templatename": 1, "state": 1, } }, ] res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] if res: data = list(res) return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 观点模板详情 @csrf_exempt def getTemplateInfo(request): _datecreated = timestamp_to_str() try: if request.method == "GET": industry1 = request.GET.get("industry1", '') # 行业 industry2 = request.GET.get("industry2", '') # 行业 orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') viewpointid = request.GET.get('viewpointid', '') viewpointname = request.GET.get('viewpointname', '') ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') templateid = request.GET.get('templateid', '') templatename = request.GET.get('templatename', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] filter_ = {"templateid": templateid} params = { "_id": 0, "id": {f"$toString": f"$_id"}, "orgid": 1, "orgname": 1, "owner": 1, "ownerid": 1, "viewpointid": 1, "viewpointname": 1, "templateid": 1, "templatename": 1, "industry1": 1, "industry2": 1, "state": 1 } res = collection.find_one(filter_, params, no_cursor_timeout=True) data = {} if res: data = res return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 新增观点模板 @csrf_exempt def newTemplateInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) industry1 = get_data.get("industry1", '') # 行业 industry2 = get_data.get("industry2", '') # 行业 industryid1 = get_data.get("industryid1", '') industryid2 = get_data.get("industryid2", '') orgid = get_data.get('orgid', '') orgname = get_data.get('orgname', '') viewpointid = get_data.get('viewpointid', '') viewpointname = get_data.get('viewpointname', '') ownerid = str(get_data.get('ownerid', '')) owner = get_data.get('owner', '') templatename = get_data.get('templatename', '') description = get_data.get('description', '') TIME = timestamp_to_str() templateid = f"{viewpointid}-t-" + get_uuid() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] data = { "industry1": industry1, "industry2": industry2, "industryid1": industryid1, "industryid2": industryid2, "orgid": orgid, "orgname": orgname, "owner": owner, "ownerid": ownerid, "viewpointid": viewpointid, "viewpointname": viewpointname, "description": description, "templateid": templateid, "templatename": templatename, "titles": [], "datecreated": TIME, "dateupdated": TIME, "sort": 100, "state": 0 } res = collection.insert_one(data) # try: # res = collection.insert_one(data) # except DuplicateKeyError: # return JsonResponse({"code": "10003", "message": f"({templatename})-此模板名称已存在"}) return JsonResponse({ "code": "0000", "message": "成功", "data": res.inserted_id, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 修改观点模板 @csrf_exempt def updateTemplateInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) industry1 = request.GET.get("industry1", '') # 行业 industry2 = request.GET.get("industry2", '') # 行业 orgid = get_data.get('orgid', '') orgname = get_data.get('orgname', '') viewpointid = get_data.get('viewpointid', '') viewpointname = get_data.get('viewpointname', '') ownerid = str(get_data.get('ownerid', '')) owner = get_data.get('owner', '') templateid = get_data.get('templateid', '') templatename = get_data.get('templatename', '') description = get_data.get('description', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] params = { "$set": { "templatename": templatename, "dateupdated": TIME, } } filter_ = {"templateid": templateid} res = collection.update_one(filter_, params) return JsonResponse({ "code": "0000", "message": "成功", "data": templateid, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 删除观点模板 @csrf_exempt def deleteTemplateInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) orgid = get_data.get('orgid', '') orgname = get_data.get('orgname', '') viewpointid = get_data.get('viewpointid', '') viewpointname = get_data.get('viewpointname', '') ownerid = str(get_data.get('ownerid', '')) owner = get_data.get('owner', '') templateid = get_data.get('templateid', '') templatename = get_data.get('templatename', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] collection_del = db_[pongo_content + '_del'] filter_ = {"templateid": templateid} data = collection.find_one(filter_) data.update({"dateupdated": _datecreated}) collection_del.insert_one(data) res = collection.delete_one(filter_) return JsonResponse({ "code": "0000", "message": "成功", "data": templateid, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 观点标题列表 @csrf_exempt def getTitleList(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": industry1 = request.GET.get("industry1", '') # 行业 industryid1 = request.GET.get("industryid1", '') # 行业 industry2 = request.GET.get("industry2", '') # 行业 industryid2 = request.GET.get("industryid2", '') # 行业 orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') viewpointid = request.GET.get('viewpointid', '') viewpointname = request.GET.get('viewpointname', '') ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] filter_ = filter_2 = {} if orgid: filter_.update({"orgid": orgid}) if ownerid: filter_.update({"ownerid": ownerid}) pipeline = [ {"$match": filter_}, { "$project": { "_id": 0, "id": {f"$toString": f"$_id"}, "orgid": 1, "orgname": 1, "owner": 1, "ownerid": 1, "viewpointid": 1, "viewpointname": 1, "templateid": 1, "templatename": 1, "titlelist": 1, "contenttemplate": 1 } }, ] res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = {} if res: data = list(res)[0] return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 观点标题详情 -- 无用 @csrf_exempt def getTitleInfo(request): _datecreated = timestamp_to_str() try: if request.method == "GET": industry1 = request.GET.get("industry1", '') # 行业 industry2 = request.GET.get("industry2", '') # 行业 orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') viewpointid = request.GET.get('viewpointid', '') viewpointname = request.GET.get('viewpointname', '') ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') templateid = request.GET.get('templateid', '') templatename = request.GET.get('templatename', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] filter_ = {"templateid": templateid} params = { "_id": 0, "id": {f"$toString": f"$_id"}, "orgid": 1, "orgname": 1, "owner": 1, "ownerid": 1, "viewpointid": 1, "viewpointname": 1, "templateid": 1, "templatename": 1, "titlelist": 1, } res = collection.find_one(filter_, params, no_cursor_timeout=True) data = {} if res: data = res return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 新增观点标题 @csrf_exempt def newTitleInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) industry1 = get_data.get('industry1', '') industryid1 = get_data.get('industryid1', '') industry2 = get_data.get('industry2', '') industryid2 = get_data.get('industryid2', '') orgid = get_data.get('orgid', '') orgname = get_data.get('orgname', '') viewpointid = get_data.get('viewpointid', '') viewpointname = get_data.get('viewpointname', '') ownerid = str(get_data.get('ownerid', '')) owner = get_data.get('owner', '') templateid = get_data.get('templateid', '') templatename = get_data.get('templatename', '') titleid = get_data.get('titleid', '') title = get_data.get('title', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] collection_t = db_[pongo_title] params_r = { "_id": 0, "id": {f"$toString": f"$_id"}, "orgid": 1, "orgname": 1, "owner": 1, "ownerid": 1, "viewpointid": 1, "viewpointname": 1, "templateid": 1, "templatename": 1, "titles": 1, } if not titleid: titleid_insert = templateid + '-' + get_uuid() title_insert = title.replace('【', '').replace('】', '') data = { "titleid": titleid_insert, "title": title_insert, "industry1": industry1, "industryid1": industryid1, "industryid2": industryid2, "industry2": industry2, "orgid": orgid, "orgname": orgname, "owner": owner, "ownerid": ownerid, "viewpointid": viewpointid, "viewpointname": viewpointname, "templateid": templateid, "templatename": templatename, "datecreated": TIME, "dateupdated": TIME, "sort": 100, "state": 0 } try: collection_t.insert_one(data) params = { "$push": {"titles": {"id": titleid_insert, "templatetitlename": title_insert, "title": title_insert, "status": 0}}, "$set": { "dateupdated": TIME, } } filter_ = {"templateid": templateid} collection.update_one(filter_, params) res = collection.find_one(filter_, params_r, no_cursor_timeout=True) return JsonResponse({ "code": "0000", "message": "成功", "data": res, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except DuplicateKeyError: return JsonResponse({"code": "10003", "message": f"{title}-已存在"}) else: code = "1000" return JsonResponse({"code": code, "message": CODE_DICT[code]}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 修改观点标题 @csrf_exempt def updateTitleInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) industry1 = get_data.get('industry1', '') industryid1 = get_data.get('industryid1', '') industry2 = get_data.get('industry2', '') industryid2 = get_data.get('industryid2', '') orgid = get_data.get('orgid', '') orgname = get_data.get('orgname', '') viewpointid = get_data.get('viewpointid', '') viewpointname = get_data.get('viewpointname', '') ownerid = str(get_data.get('ownerid', '')) owner = get_data.get('owner', '') templateid = get_data.get('templateid', '') templatename = get_data.get('templatename', '') titleid = get_data.get('titleid', '') title = get_data.get('title', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] collection_t = db_[pongo_title] params_r = { "_id": 0, "id": {f"$toString": f"$_id"}, "orgid": 1, "orgname": 1, "owner": 1, "ownerid": 1, "viewpointid": 1, "viewpointname": 1, "templateid": 1, "templatename": 1, "titles": 1, } if not titleid: titleid_insert = templateid+'-'+get_uuid() title_insert = title.replace('【', '').replace('】', '') data = { "titleid": titleid_insert, "title": title_insert, "industry1": industry1, "industryid1": industryid1, "industryid2": industryid2, "industry2": industry2, "orgid": orgid, "orgname": orgname, "owner": owner, "ownerid": ownerid, "viewpointid": viewpointid, "viewpointname": viewpointname, "templateid": templateid, "templatename": templatename, "datecreated": TIME, "dateupdated": TIME, "sort": 100, "state": 0 } try: collection_t.insert_one(data) params = { "$push": {"titles": {"id": titleid_insert, "templatetitlename": title_insert, "title": title_insert, "status": 0}}, "$set": { "dateupdated": TIME, } } filter_ = {"templateid": templateid} collection.update_one(filter_, params) res = collection.find_one(filter_, params_r, no_cursor_timeout=True) return JsonResponse({ "code": "0000", "message": "成功", "data": res, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except DuplicateKeyError: return JsonResponse({"code": "10003", "message": f"{title}-已存在"}) else: filter_ = {"templateid": templateid} titleid_insert = templateid + '-' + get_uuid() title_insert = title.replace('【', '').replace('】', '') data = { "title": title_insert, "titleid": titleid_insert, "industry1": industry1, "industryid1": industryid1, "industryid2": industryid2, "industry2": industry2, "orgid": orgid, "orgname": orgname, "owner": owner, "ownerid": ownerid, "viewpointid": viewpointid, "viewpointname": viewpointname, "templateid": templateid, "templatename": templatename, "datecreated": TIME, "dateupdated": TIME, "sort": 100, "state": 0 } try: collection_t.insert_one(data) res = collection.find_one(filter_, params_r, no_cursor_timeout=True) titles = res.get('titles', []) titlelist_new = [] for item in titles: titleid_old = item.get('id', '') title_old = item.get('title', '') if titleid_old == titleid: item.update({"templatetitlename": title_insert, "title": title_insert}) titlelist_new.append(item) filter_t = {"title": title_old} collection_t.delete_one(filter_t) else: titlelist_new.append(item) params = { "$set": { "titles": titlelist_new, "dateupdated": TIME, } } collection.update_one(filter_, params) # res = collection.find_one_and_update(filter_, params, params_r) res = collection.find_one(filter_, params_r, no_cursor_timeout=True) return JsonResponse({ "code": "0000", "message": "成功", "data": res, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except DuplicateKeyError: return JsonResponse({"code": "10003", "message": f"{title}-已存在"}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 删除观点标题 @csrf_exempt def deleteTitleInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) industry1 = get_data.get('industry1', '') industry2 = get_data.get('industry2', '') orgid = get_data.get('orgid', '') orgname = get_data.get('orgname', '') viewpointid = get_data.get('viewpointid', '') viewpointname = get_data.get('viewpointname', '') ownerid = str(get_data.get('ownerid', '')) owner = get_data.get('owner', '') templateid = get_data.get('templateid', '') templatename = get_data.get('templatename', '') title = get_data.get('title', '') titleid = get_data.get('titleid', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] collection_t = db_[pongo_title] filter_ = {"templateid": templateid} params_r = { "_id": 0, "id": {f"$toString": f"$_id"}, "orgid": 1, "orgname": 1, "owner": 1, "ownerid": 1, "viewpointid": 1, "viewpointname": 1, "templateid": 1, "templatename": 1, "titles": 1, } res_data = collection.find_one(filter_, params_r, no_cursor_timeout=True) titles = res_data.get('titles', []) titlelist_new = [] for item in titles: titleid_old = item.get('id', '') title_old = item.get('templatetitlename', '') title_name = item.get('title', '') if titleid_old != titleid: titlelist_new.append(item) else: filter_t = {"title": title_old} collection_t.delete_one(filter_t) collection_t_del = db_[pongo_title+'_del'] collection_t_del.insert_one({"id": titleid_old, "templatetitlename": title_old, "title": title_name}) params = { "$set": { "dateupdated": TIME, "titles": titlelist_new } } filter_ = {"templateid": templateid} collection.update_one(filter_, params) res = collection.find_one(filter_, params_r, no_cursor_timeout=True) return JsonResponse({ "code": "0000", "message": "成功", "data": res, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 判断观点标题是否重复 @csrf_exempt def judgeTitleInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) industry1 = get_data.get('industry1', '') industry2 = get_data.get('industry2', '') orgid = get_data.get('orgid', '') orgname = get_data.get('orgname', '') viewpointid = get_data.get('viewpointid', '') viewpointname = get_data.get('viewpointname', '') ownerid = str(get_data.get('ownerid', '')) owner = get_data.get('owner', '') templateid = get_data.get('templateid', '') templatename = get_data.get('templatename', '') title = get_data.get('title', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_title] data = { "title": title, "orgid": orgid, "orgname": orgname, "owner": owner, "ownerid": ownerid, "viewpointid": viewpointid, "viewpointname": viewpointname, "templateid": templateid, "templatename": templatename, "datecreated": TIME, "dateupdated": TIME, "sort": 100, "state": 0 } try: collection.insert_one(data) except DuplicateKeyError: return JsonResponse({"code": "10003", "message": f"{title}-已存在"}) return JsonResponse({ "code": "0000", "message": "成功", "data": templateid, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # ########################################################################## # 词分类管理 # ######################################################################### # 词分类列表 @csrf_exempt def simian_wordgroup_findall(request): table1 = f"{simian_wordgroup_table} wt" table2 = f"{simian_wordgroup_viewpoint_table} wv" if request.method == 'POST': get_data = {} try: with LogMysqlApeT(ape_db) as m_client: get_data = json.loads(request.body) industry1 = get_data.get("industry1", '') # 行业 owner = get_data.get("owner", '') # 所属人 ownerid = get_data.get("ownerid", '') # 所属人id orgname = get_data.get("orgname", '') # 所属人机构 orgid = get_data.get("orgid", '') # 所属人机构id industry2 = get_data.get("industry2", '') # 二级行业 industryid1 = get_data.get("industryid1", '') # 一级行业id industryid2 = get_data.get("industryid2", '') # 二级行业id wordgroupname = get_data.get("wordgroupname", '') # 词名称 page = get_data.get("page", 1) # 页数 size = get_data.get("size", 20) # 条数 condition = f"""select wt.id, wt.wordgroupname ,wt.industry,wt.industry2,wt.industryid1,wt.industryid2, wt.orgid,wt.orgname,wt.ownerid,wt.owner, wv.tagmust, wv.owner, wv.viewpoint,wv.tagcity,wv.tagbrand, wt.status, wv.tagscience, wv.taglocation, wt.description from {table1} left join {table2} ON wt.id = wv.wgid where wt.status=0 """ count = f"""select count(id) as total from {table1} where 1=1 and status=0 """ if orgid: condition += " and wt.orgid='{}' ".format(orgid) count += " and orgid='{}' ".format(orgid) # if ownerid: # condition += " and wt.ownerid='{}' ".format(ownerid) # count += " and ownerid='{}' ".format(ownerid) if industryid1: condition += " and wt.industryid1='{}' ".format(industryid1) count += " and industryid1='{}' ".format(industryid1) if industryid2: condition += " and wt.industryid2='{}' ".format(industryid2) count += " and industryid2='{}' ".format(industryid2) if wordgroupname: condition += " and wt.wordgroupname='{}' ".format(wordgroupname) count += " and wordgroupname='{}' ".format(wordgroupname) condition += " order by wt.industry,wt.wordgroupname limit {},{}".format((page - 1) * size, size) logger.info(condition) datalist = m_client.search_db(condition) total = m_client.search_db(count) data = {"datalist": datalist, "total": total[0]} logger.info('# 获取词分类数据 #') code = "0000" return JsonResponse({"code": code, "data": data, "message": CODE_DICT[code]}) # else: # code = "1019" # return JsonResponse({"code": code, "message": CODE_DICT[code]}) else: code = "1016" return JsonResponse({"code": code, "message": CODE_DICT[code]}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 词分类配置查询 @csrf_exempt def simian_wordgroup_viewpoint_deatils(request): if request.method == 'POST': get_data = {} try: with LogMysqlApeT(ape_db) as m_client: get_data = json.loads(request.body) industry1 = get_data.get("industry1", '') # 行业 owner = get_data.get("owner", '') # 所属人 ownerid = get_data.get("ownerid", '') # 所属人id orgname = get_data.get("orgname", '') # 所属人机构 orgid = get_data.get("orgid", '') # 所属人机构id industry2 = get_data.get("industry2", '') # 二级行业 wordgroupname = get_data.get("wordgroupname", '') # 词名称 wgid = get_data.get("wgid", '') # 词Id industryid1 = get_data.get("industryid1", '') # 一级行业id industryid2 = get_data.get("industryid2", '') # 二级行业id condition = f"""select a.*,b.* FROM ( select svt.id as viewpointid, svt.viewpoint,svt.industry2,svt.industryid2,svt.industryid1, svt.orgid,svt.orgname,svt.ownerid,svt.owner FROM simian_viewpoint_table svt , (SELECT industry2,industryid2,orgid,ownerid FROM simian_wordgroup_table WHERE id = '{wgid}') swt WHERE svt.industryid2 = swt.industryid2 and svt.orgid = swt.orgid ) a LEFT JOIN (SELECT wvt.id, wt.id as wgid, wt.wordgroupname, wt.industry, wt.owner, wvt.tagmust, wvt.taglocation, wt.description,wvt.viewpointuuid, wvt.tagcity,wvt.tagbrand,wvt.tagscience FROM simian_wordgroup_viewpoint_table wvt, simian_wordgroup_table wt WHERE wvt.wgid = wt.id AND wvt.wgid = '{wgid}') b ON a.viewpointid = b.viewpointuuid ORDER BY b.id DESC,a.viewpointid""" data = m_client.search_db(condition) code = "0000" return JsonResponse({"code": code, "data": data, "message": CODE_DICT[code]}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 词分类观点关系保存 @csrf_exempt def simian_wordgroup_viewpoint_add(request): table = "simian_wordgroup_viewpoint_table" if request.method == 'POST': get_data = {} try: with LogMysqlApeT(ape_db) as m_client: data = json.loads(request.body) if data: wgid = data[0].get("wgid", '') delsql = "delete from {} where wgid = '{}'".format(table, wgid) m_client.del_db(delsql) TIME = timestamp_to_str(time.time()) for item in range(len(data)): condition = { "id": "simian-wg-vp-" + get_uuid(), "wgid": data[item]['wgid'], "tagmust": data[item]['tagmust'], "tagcity": data[item]['tagcity'], "tagbrand": data[item]['tagbrand'], "tagscience": data[item]['tagscience'], "wordgroupname": data[item]['wordgroupname'], "owner": data[item]['owner'], "ownerid": data[item]['ownerid'], "orgid": data[item]['orgid'], "orgname": data[item]['orgname'], "industry": data[item]['industry'], "industry2": data[item]['industry2'], "industryid1": data[item]['industryid1'], "industryid2": data[item]['industryid2'], "viewpoint": data[item]['viewpoint'], "viewpointuuid": data[item]['viewpointid'], "industryviewpointmd5": md5_me(data[item]['industry']) + '-' + md5_me(data[item]['viewpoint']), "createtime": TIME, "updatetime": TIME, "status": 0, "taglocation": data[item]['taglocation'] } logger.info(str(condition)) flag = m_client.updata_db(condition, table) else: code = "1020" return JsonResponse({"code": code, "data": "", "message": CODE_DICT[code]}) code = "0000" return JsonResponse({"code": code, "data": flag, "message": CODE_DICT[code]}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # #################################################################### # 请求脚本 # #################################################################### # 将观点模板同步 @csrf_exempt def oldTemplageInsertNew(request): _datecreated = timestamp_to_str() try: if request.method == "GET": with LogMysqlApeT(ape_db) as m_client: sql = f""" SELECT * FROM simian_vpcontent_table WHERE property='母模板' AND deleted='0' """ data_list = m_client.search_db(sql) with MongoDBClient() as mg_client: db_ = mg_client[db] # 获取collection collection = db_[pongo_content] for item in data_list: industry1 = item.get('industry', '') industry2 = item.get('industry2', '') industryid1 = '' industryid2 = '' if industry2 == '成人英语': industryid1 = '1001' industryid2 = '10011002' if industry2 == '留学': industryid1 = '1002' industryid2 = '10021011' if industry2 == '雅思': industryid1 = '1000' industryid2 = '10001001' if industry2 == '托福': industryid1 = '1000' industryid2 = '10001000' templateid = item.get('id') content = item.get('viewpointcontent', '') templatename = item.get('parentname', '') viewpointid = item.get('viewpointuuid', '') viewpointname = item.get('viewpoint', '') data = { "wordgroups": [], "content": content, "industry1": industry1, "industry2": industry2, "industryid1": industryid1, "industryid2": industryid2, "orgid": "bae479060ab74020a085ad665921e8f5", "orgname": "环球雅思", "owner": "", "ownerid": "", "viewpointid": viewpointid, "viewpointname": viewpointname, "description": "观点的写作方向", "templateid": templateid, "templatename": templatename, "titles": [], "datecreated": _datecreated, "dateupdated": _datecreated, "sort": 100, "state": 0 } collection.insert_one(data) return JsonResponse({ "code": "0000", "message": "成功", "data": templateid, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 将观点标题同步 @csrf_exempt def oldTitleTemplageInsertNew(request): _datecreated = timestamp_to_str() try: if request.method == "GET": with LogMysqlApeT(ape_db) as m_client: sql = f""" SELECT * FROM simian_vptitle_table WHERE property='母标题' AND deleted='0' """ data_list_res = m_client.search_db(sql) data_dict = {} sql1 = f""" SELECT contentid, viewpointuuid FROM simian_vptitle_table WHERE property='子标题' AND deleted='0' GROUP BY viewpointuuid ORDER BY contentid DESC """ data_list_res_1 = m_client.search_db(sql1) c_v_dict = v_c_dict = {} for j in data_list_res_1: c_v_dict.update({j.get('viewpointuuid'): j.get('contentid')}) v_c_dict.update({j.get('contentid'): j.get('viewpointuuid')}) for i in data_list_res: contentid_templateid = i.get('contentid') viewpointuuid = i.get('viewpointuuid') if not contentid_templateid: contentid_templateid = c_v_dict.get(viewpointuuid, '0') title = i.get('viewpointtitle') tid = i.get('id') if viewpointuuid == v_c_dict.get(contentid_templateid, ''): data_dict.setdefault(contentid_templateid, []).append({ "id": tid, "templatetitlename": title, "title": title, }) with MongoDBClient() as mg_client: db_ = mg_client[db] # 获取collection collection = db_[pongo_content] for k, v in data_dict.items(): params = { "$set": { # "dateupdated": TIME, "titles": v } } filter_ = {"templateid": k} collection.update_one(filter_, params) return JsonResponse({ "code": "0000", "message": "成功", "data": data_dict.get("0", ''), "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # #################################################################### # 内容生成 # #################################################################### # 内容生成接口 # 调取此接口,期间不能够进行修改,直接生成内容 @csrf_exempt def produceContentTitle(request): _datecreated = timestamp_to_str() if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) templateid = get_data.get("templateid", '') # templateid logger.info('进行内容和标题的替换生成') logger.info('接收到数据:%s' % str(get_data)) with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] params = { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "state": 1 } filter_ = {"templateid": templateid} print(filter_) data_res = collection.find_one(filter_, params, no_cursor_timeout=True) state = data_res.get('state', '') if state == 0: logger.info(f'可以生成替换文') # 将任务塞入redis队列 r_client = Redis() r_client.conn.lpush(pongo_produce_vpc_vpt, templateid) params = { "$set": { "state": 1, "dateupdated": _datecreated, } } filter_ = {"templateid": templateid} collection.update_one(filter_, params) code = "0000" else: code = "1023" return JsonResponse({"code": code, "data": "ok", "message": CODE_DICT[code], "datecreated": _datecreated}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated}) return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated}) # 删除生成的子内容 @csrf_exempt def deleteReContentTitle(request): _datecreated = timestamp_to_str() if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) templateid = get_data.get("templateid", '') # templateid logger.info('删除生成的子内容') logger.info('接收到数据:%s' % str(get_data)) with MongoDBClient() as mg_client, LogMysqlApeT(ape_db) as m_client: db_ = mg_client[db] # 获取collection collection = db_[pongo_content] params = { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "state": 1 } filter_ = {"templateid": templateid} print(filter_) data_res = collection.find_one(filter_, params, no_cursor_timeout=True) state = data_res.get('state', '') if state == 3: logger.info(f'可以删除') # 删除子内容 sql_deleted_vptitle = f""" delete from {simian_vptitle_table} where contentid='{templateid}' and used=0 and deleted=0 and property='子标题' """ m_client.del_db(sql_deleted_vptitle) logger.info(sql_deleted_vptitle) sql_deleted_vpcontent = f"""delete from {simian_vpcontent_table} where belongtemplate='{templateid}' and deleted=0 and used=0 and property='子模板' """ m_client.del_db(sql_deleted_vpcontent) logger.info(sql_deleted_vpcontent) # 删除成功后修改值,可以进行编辑 params = { "$set": { "state": 0, "dateupdated": _datecreated, } } filter_ = {"templateid": templateid} collection.update_one(filter_, params) code = "0000" else: code = "1024" return JsonResponse({"code": code, "data": "ok", "message": CODE_DICT[code], "datecreated": _datecreated}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated}) return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated}) # #################################################################### # 关键词统计 # #################################################################### # 关键词统计列表 @csrf_exempt def keywordStatisticsList(request): _datecreated = timestamp_to_str() logger.info("keywordStatisticsList") try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "0000", "message": "success"}) if request.method == "GET": ownerid = str(request.GET.get('ownerid', '')) owner = request.GET.get('owner', '') keyword = request.GET.get('keyword', '') area = request.GET.get('area', '') wordgroup = request.GET.get('wordgroup', '') wordgroupid = request.GET.get('wordgroupid', '') orgid = request.GET.get('orgid', '') orgname = request.GET.get('orgname', '') industry1 = request.GET.get('industry1', '') industryid1 = request.GET.get('industryid1', '') industry2 = request.GET.get('industry2', '') industryid2 = request.GET.get('industryid2', '') onlinedate = request.GET.get('onlinedate', '') date = request.GET.get('date', '["", ""]') date = json.loads(date) logger.info(str(date)) logger.info(str(type(date))) status = str(request.GET.get('status', '')) page = int(request.GET.get('page', 1)) size = int(request.GET.get('size', 10)) with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_keyword] filter_ = {} if orgid: filter_.update({"orgid": orgid}) if industryid1: filter_.update({"industryid1": industryid1}) if industryid2: filter_.update({"industryid2": industryid2}) if keyword: filter_.update({"keyword": {"$regex": keyword}}) # 0 待编辑 # 1 已编辑 # 2 已发布 # 10 计划中 if status == "0": filter_.update({ "publishlist": {"$elemMatch": {"status": 0}}, # "publishlist.status": 0 }) if status == "1": filter_.update({ "publishlist": {"$elemMatch": {"status": 1}}, # "publishlist.status": 1 }) if status == "2": filter_.update({ "publishlist": {"$elemMatch": {"status": 2}}, # "publishlist.status": 2 }) if status == "10": filter_.update({ "publishlist": {"$elemMatch": {"status": 10}}, # "publishlist.status": 10 }) if date[0] != "" and date: tk = [ {"onlinedate": {"$gte": date[0]}}, {"onlinedate": {"$lte": date[1]}} ] if status == "0": tk.append({"status": 0}) if status == "1": tk.append({"status": 1}) if status == "2": tk.append({"status": 2}) if status == "10": tk.append({"status": 10}) tem = {"publishlist": {"$elemMatch": {"$and": tk}}} filter_.update(tem) pipeline = [ {"$match": filter_}, {"$sort": {"datecreated": -1, "keyword": 1}}, {"$skip": (page - 1) * size}, {"$limit": size}, # {"$limit": page * size}, # {"$skip": (page - 1) * size}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "wordgroup": 1, "wordgroupid": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "keyword": 1, "area": 1, "orgid": 1, "orgname": 1, "publishlist": 1, "publishsum": {f"$size": f"$publishlist"}, # 发布任务总数量 "published": { "$function": { "body": """function(name) { var count = 0; for(j = 0,len=name.length; j< len; j++) { if(name[j].status === 2){ count += 1; } } return count ; } """, "args": ["$publishlist"], "lang": "js" }, # 已发布数量 }, "toeditpublish": { "$function": { "body": """function(name) { var count = 0; for(j = 0,len=name.length; j< len; j++) { if(name[j].status === 0){ count += 1; } } return count ; } """, "args": ["$publishlist"], "lang": "js" }, # 待编辑数量 }, "editpublish": { "$function": { "body": """function(name) { var count = 0; for(j = 0,len=name.length; j< len; j++) { if(name[j].status === 1){ count += 1; } } return count ; } """, "args": ["$publishlist"], "lang": "js" }, # 已编辑数量 }, "prepublish": { "$function": { "body": """function(name) { var count = 0; for(j = 0,len=name.length; j< len; j++) { if(name[j].status === 10){ count += 1; } } return count ; } """, "args": ["$publishlist"], "lang": "js" }, # 计划中数量 }, }, }, ] logger.info(pipeline) total = collection.count_documents(filter=filter_) res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] if res: data = list(res) else: code = "1016" return JsonResponse({"code": code, "message": CODE_DICT[code]}) return JsonResponse({ "code": "0000", "message": "成功", "total": total, "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) return JsonResponse({"code": "1001", "message": "no this method"}) # 关键词统计标题详情 @csrf_exempt def getKeywordStatisticsInfo(request): _datecreated = timestamp_to_str() try: if request.method == "GET": keywordid = request.GET.get('keywordid', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_keyword] filter_ = {"_id": ObjectId(keywordid)} params = { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "wordgroupid": 1, "wordgroup": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "keyword": 1, "area": 1, "orgid": 1, "orgname": 1, "publishlist": 1, } print(filter_) # res = collection.find_one(filter_, no_cursor_timeout=True) res = collection.find_one(filter_, params, no_cursor_timeout=True) data = {} if res: data = res return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated}) return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated}) # 关键词统计编辑 @csrf_exempt def updateKeywordStatisticsInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": "1002", "message": str(e)}) keywordid = get_data.get('keywordid', '') tid = get_data.get('tid', '') aid = int(get_data.get('aid', 0)) title = get_data.get('title', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_keyword] collection_t = db_[pongo_tdk_title] params_r = { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "wordgroupid": 1, "wordgroup": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "keyword": 1, "area": 1, "orgid": 1, "orgname": 1, "publishlist": 1, } # filter_ = {"_id": ObjectId(keywordid)} # print(filter_) # data_res = collection.find_one(filter_, params, no_cursor_timeout=True) # publishlist = data_res.get("publishlist", []) # publishlist_new = [] data_title = { "tid": tid, "aid": aid, "title": title, "dateupdated": TIME, "datecreated": TIME, } try: collection_t.insert_one(data_title) # for item in publishlist: # item_aid = int(item.get("aid", 0)) # if item_aid == aid: # item.update({"title": title, "status": 1}) # publishlist_new.append(item) xgz = {} xgz.update({"dateupdated": TIME, "publishlist.$.title": title, "publishlist.$.status": 1, "status": 1}) params = { "$set": xgz, } filter_ = {"_id": ObjectId(keywordid), "publishlist": {"$elemMatch": {"id": tid}}} collection.update_one(filter_, params) filter_ = {"_id": ObjectId(keywordid)} print(filter_) data_res = collection.find_one(filter_, params_r, no_cursor_timeout=True) publishlist_new = data_res.get("publishlist", []) code = "0000" return JsonResponse({ "code": code, "message": CODE_DICT[code], "data": publishlist_new, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except DuplicateKeyError: code = "1022" return JsonResponse({"code": code, "message": f"({title})-{CODE_DICT[code]}"}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated}) return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated}) # 关键词统计获取覆盖词函数 def get_cover_word_list(get_data): title = get_data.get("title", '') orgid = get_data.get("orgid", '') industryid1 = get_data.get("industryid1", '') industryid2 = get_data.get("industryid2", '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_keyword] params = { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "keyword": 1, } filter_ = {"orgid": orgid, "industryid1": industryid1, "industryid2": industryid2} print(filter_) data_res = collection.find(filter_, params, no_cursor_timeout=True) #sql = "select keyword from {}".format(ape_keyword_table) #res = m_client.search_db(sql) keyword_list = [item_.get('keyword') for item_ in data_res] res_list = [] for item in keyword_list: # keyword = item index = 0 if item: for keyword_item in item: index = title.find(keyword_item, index) print("index", index) if index < 0: break else: res_list.append(item) data = list(set(list(filter(lambda x: len(x) > 0, res_list)))) if not data: data = ['没有覆盖的关键词'] return data # 关键词统计获取取当前标题的词覆盖有哪些词 @csrf_exempt def getKeywordCover(request): _datecreated = timestamp_to_str() if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) title = get_data.get("title", '') orgid = get_data.get("orgid", '') industryid1 = get_data.get("industryid1", '') industryid2 = get_data.get("industryid2", '') logger.info('获取取当前标题的词覆盖有哪些词') logger.info('接收到数据:%s' % str(get_data)) data = get_cover_word_list(get_data) code = "0000" return JsonResponse({"code": code, "data": data, "datecreated": _datecreated, "message": CODE_DICT[code]}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated}) return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated}) # 获取当前词所在分组的词有哪些 @csrf_exempt def getWordgroupKeyword(request): _datecreated = timestamp_to_str() if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) keyword = get_data.get("keyword", '') # keyword kid = get_data.get("kid", '') # kid orgid = get_data.get("orgid", '') # kid wordgroup = get_data.get("wordgroup", '') # kid wordgroupid = get_data.get("wordgroupid", '') # kid industry1 = get_data.get("industry1", '') # 行业1 industry2 = get_data.get("industry2", '') # 行业2 industryid1 = get_data.get("industryid1", '') # 行业id1 industryid2 = get_data.get("industryid2", '') # 行业id2 logger.info('获取当前词所在分组的词有哪些') logger.info('接收到数据:%s' % str(get_data)) with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_keyword] params = { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "keyword": 1, } filter_ = {"orgid": orgid, "industryid1": industryid1, "industryid2": industryid2, "wordgroupid": wordgroupid} print(filter_) data_res = collection.find(filter_, params, no_cursor_timeout=True) keyword_list = [item_.get('keyword') for item_ in data_res] code = "0000" return JsonResponse({"code": code, "data": keyword_list, "message": CODE_DICT[code], "datecreated": _datecreated}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated}) return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated}) # ########################### # 站点接口 # ########################### # 获取站点列表 @csrf_exempt def getAllianceList(request): _datecreated = timestamp_to_str() if request.method == 'GET': try: with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_alliance] params = { # _id 可以设置0 别的参数不可以 "_id": 0, "aid": 1, "aname": 1, "id": {f"$toString": f"$_id"}, "labellist": 1, } filter_ = {} print(filter_) data_res = collection.find(filter_, params, no_cursor_timeout=True) data = list(data_res) code = "0000" return JsonResponse({"code": code, "data": data, "message": CODE_DICT[code], "datecreated": _datecreated}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated}) return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated}) # ########################### # 发布接口 # ########################### # # 发布配置 @csrf_exempt def publishTaskCreate(request): _datecreated = timestamp_to_str() if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) date = get_data.get("date", []) aid = int(get_data.get("aid", 0)) aname = get_data.get("aname", '') tcid = get_data.get("id", '') distribute = int(get_data.get("distribute", '0')) # 1 为话题 2 为关键词 order = int(get_data.get("order", '0')) # 1 为词覆盖 2 为指数 info = get_data.get("info", []) # publishnum = get_data.get("publishnum", 0) # industry1 = get_data.get("industry1", '') # industryid1 = get_data.get("industryid1", '') # industry2 = get_data.get("industry2", '') # industryid2 = get_data.get("industryid2", '') date_list = dateRange(date[0], date[1]) logger.info('发布配置') logger.info('接收到数据:%s' % str(get_data)) with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_publish_task] if tcid: filter_ = {"_id": ObjectId(tcid)} res = collection.find_one(filter_) else: res = {} if res: data = { "date": date, "datelist": date_list, "distribute": distribute, "order": order, "info": info, } data.update( {"dateupdated": _datecreated} ) params_ = { "$set": data, } collection.update_one(filter_, params_) else: data = { "date": date, "datelist": date_list, "aid": aid, "aname": aname, "distribute": distribute, "order": order, "info": info, "state": 0, "datecreated": _datecreated, "dateupdated": _datecreated, } collection.insert_one(data) code = "0000" return JsonResponse({"code": code, "data": "ok", "message": CODE_DICT[code], "datecreated": _datecreated}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated}) return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated}) # # 发布配置详情 @csrf_exempt def publishTaskInfo(request): _datecreated = timestamp_to_str() if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) aid = int(get_data.get("aid", 0)) state = int(get_data.get("state", 10000)) page = int(get_data.get("page", 1)) size = int(get_data.get("size", 10)) logger.info('发布配置详情') logger.info('接收到数据:%s' % str(get_data)) with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_publish_task] filter_ = {"aid": aid} if state != 10000: filter_.update({"state": state}) pipeline = [ {"$match": filter_}, {"$sort": {"state": 1, "datecreated": -1}}, {"$skip": (page - 1) * size}, {"$limit": size}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "id": {f"$toString": f"$_id"}, "date": 1, "aid": 1, "aname": 1, "distribute": 1, "order": 1, "state": 1, # 如果内容正在生成或者生成,不让修改 "labellist": 1, "info": 1, "datelist": 1, } }, ] print(pipeline) total = collection.count_documents(filter=filter_) res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] if res: data = list(res) code = "0000" return JsonResponse({"code": code, "data": data, "message": CODE_DICT[code], "datecreated": _datecreated, "total": total}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated}) return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated}) # # 发布任务生成给词做标记,给标题做预处理 @csrf_exempt def publishTaskProduce(request): _datecreated = timestamp_to_str() logger.info(f'# 发布任务生成给词做标记,给标题做预处理') if request.method == 'GET': try: date = request.GET.get("date", '') logger.info('发布任务生成给词做标记,给标题做预处理') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_publish_task] collection_k = db_[pongo_keyword] filter_ = {"datelist": {"$elemMatch": {"$eq": date}}, "state": 0} pipeline = [ {"$match": filter_}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "id": {f"$toString": f"$_id"}, "date": 1, "datelist": 1, "aid": 1, "aname": 1, "distribute": 1, "order": 1, "state": 1, # 如果内容正在生成或者生成,不让修改 "info": 1, } }, ] logger.info(pipeline) # total = collection.count_documents(filter=filter_) res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] if res: logger.info(f'开始遍历配置的发布任务') for item in res: t_id = item.get('id') info = item.get('info', []) aid = int(item.get('aid', 0)) aname = item.get('aname', '') distribute = item.get('distribute', '') order = item.get('order', '') datelist = item.get('datelist', []) logger.info(f"datelist - {str(datelist)}") keyword_count = {} for i_date in datelist: logger.info(i_date) for i_info in info: labelid1 = i_info.get('labelid1', '') labelid2 = i_info.get('labelid2', '') labelname1 = i_info.get('labelname1', '') labelname2 = i_info.get('labelname2', '') count = int(i_info.get('count', 0)) filter_k = { "state": 0, "publishlist": {"$elemMatch": {"labelname1": labelname1, "labelname2": labelname2}}, } logger.info(f"找出条件为 --- {str(filter_k)} --- 的词") pipeline_k = [ {"$match": filter_k}, # {"$limit": count}, {"$sample": {"size": count}}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "id": {f"$toString": f"$_id"}, "keyword": 1, "publishlist": 1, } }, ] total = collection_k.count_documents(filter=filter_k) tem = {"total": total, "filter_k": str(filter_k)} keyword_count.setdefault(i_date, []).append(tem) logger.info(f'查询数据个数为:{total}') res_k = collection_k.aggregate(pipeline=pipeline_k, allowDiskUse=True) for k_item in res_k: k_id = k_item.get('id') # 标题 publishlist = k_item.get('publishlist', []) publishlist_new = [] for p_item in publishlist: p_aid = int(p_item.get('aid', 0)) if p_aid: if int(aid) == p_aid: p_item.update({"title": "", "status": 0, "onlinedate": i_date}) publishlist_new.append(p_item) # 将状态值变为 state变为 10 filter_k = {"_id": ObjectId(k_id)} params_k = { "$set": { "dateupdated": _datecreated, "publishlist": publishlist_new, "state": 10, "prepstatus": 1, "pstatus": 0 }, } collection_k.update_one(filter_k, params_k) pass filter_t = {"_id": ObjectId(t_id)} params_t = { "$set": { "dateupdated": _datecreated, "state": 1, "keyword_count": keyword_count, }, } collection.update_one(filter_t, params_t) code = "0000" return JsonResponse({"code": code, "data": data, "message": CODE_DICT[code], "datecreated": _datecreated}) except Exception as e: return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated}) return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated})
# _*_ coding: utf-8 _*_ from django.shortcuts import render, HttpResponse from django.http import JsonResponse from django.views.decorators.csrf import csrf_exempt, csrf_protect # Create your views here. import os import oss2 import uuid import sys import ast import time import json import redis import hashlib import requests import random from tools.mysql_client import LogMysqlApe, LogMysql, LogMysqlApeT, LogMysqlAlliance from tools.excel_client import read_excel from upload_pro.settings import logger from urllib.parse import urlparse, unquote from django.conf import settings from tools.mongodb_client import MongoDBClient from bson import ObjectId from bson import json_util from bson.json_util import dumps def get_uuid(): res = str(uuid.uuid4()) UUID = ''.join(res.split('-')) return UUID def md5_me(key): md5 = hashlib.md5() md5.update(str(key).encode('utf-8')) value = md5.hexdigest() return value # 把格式化时间转换成时间戳 def str_to_timestamp(str_time=None, format='%Y-%m-%d %H:%M:%S'): if str_time: time_tuple = time.strptime(str_time, format) # 把格式化好的时间转换成元祖 result = time.mktime(time_tuple) # 把时间元祖转换成时间戳 return int(result) return int(time.time()) # 把时间戳转换成格式化 def timestamp_to_str(timestamp=None, format='%Y-%m-%d %H:%M:%S'): if timestamp: time_tuple = time.localtime(timestamp) # 把时间戳转换成时间元祖 result = time.strftime(format, time_tuple) # 把时间元祖转换成格式化好的时间 return result else: return time.strftime(format) db = settings.PONGO_DB pongo_wordgroup = "pongo_wordgroup" pongo_content = "pongo_content" pongo_category = "pongo_category" # 词组列表 @csrf_exempt def getWordgroupList(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) return JsonResponse({"code": 0, "message": "success"}) if request.method == "GET": ownerid = request.GET.get('ownerid', '') owner = request.GET.get('owner', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_wordgroup] filter_ = {} if ownerid: filter_ = {"ownerid": ownerid} if owner: filter_ = {"owner": owner} pipeline = [ {"$match": filter_}, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "name": 1, "owner": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "itemnum": {f"$size": f"$items"}, } }, ] print(pipeline) res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] if res: data = list(res) return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) return JsonResponse({"code": 1001, "message": "no this method"}) # 词组详情 @csrf_exempt def getWordgroupInfo(request): _datecreated = timestamp_to_str() try: if request.method == "GET": wordgroupid = request.GET.get('wordgroupid', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_wordgroup] filter_ = {"_id": ObjectId(wordgroupid)} params = { "_id": 0, "id": {f"$toString": f"$_id"}, "name": 1, "items": 1, "owner": 1, "ownerid": 1, } print(filter_) # res = collection.find_one(filter_, no_cursor_timeout=True) res = collection.find_one(filter_, params, no_cursor_timeout=True) data = [] if res: data = list(res) return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) return JsonResponse({"code": 1001, "message": "no this method"}) # 新增词组 @csrf_exempt def newWordgroupInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) name = get_data.get('name', '') items = get_data.get('items', []) owner = get_data.get('owner', '') ownerid = get_data.get('ownerid', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_wordgroup] data = { "name": name, "items": items, "owner": owner, "ownerid": ownerid, "datecreated": TIME, "dateupdated": TIME, "sort": 10, "state": 0 } res = collection.insert_one(data) return JsonResponse({ "code": "0000", "message": "成功", "data": res.inserted_id, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) return JsonResponse({"code": 1001, "message": "no this method"}) # 修改词组 @csrf_exempt def updateWordgroupInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) name = get_data.get('name', '') items = get_data.get('items', []) wordgroupid = get_data.get('wordgroupid', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_wordgroup] params = { "$set": { "name": name, "items": items, "dateupdated": TIME, }, } filter_ = {"_id": ObjectId(wordgroupid)} res = collection.update_one(filter_, params) return JsonResponse({ "code": "0000", "message": "成功", "data": wordgroupid, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) return JsonResponse({"code": 1001, "message": "no this method"}) # 删除词组 @csrf_exempt def deleteWordgroupInfo(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) name = get_data.get('name', '') items = get_data.get('items', []) wordgroupid = get_data.get('wordgroupid', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_wordgroup] collection_del = db_[pongo_wordgroup+'_del'] filter_ = {"_id": ObjectId(wordgroupid)} data = collection.find_one(filter_) data.update({"dateupdated": _datecreated}) collection_del.insert_one(data) res = collection.delete_one(filter_) return JsonResponse({ "code": "0000", "message": "成功", "data": wordgroupid, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) return JsonResponse({"code": 1001, "message": "no this method"}) # 文章模板列表 @csrf_exempt def getContentList(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) return JsonResponse({"code": 0, "message": "success"}) if request.method == "GET": ownerid = request.GET.get('ownerid', '') owner = request.GET.get('owner', '') categoryid = request.GET.get('categoryid', '') category = request.GET.get('category', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] filter_ = filter_2 = {} if ownerid: filter_ = {"ownerid": ownerid} if owner: filter_ = {"owner": owner} if categoryid: filter_2 = {"categoryid": categoryid} if category: filter_2 = {"category": category} filter_.update(filter_2) pipeline = [ {"$match": filter_}, { "$project": { "_id": 0, "id": {f"$toString": f"$_id"}, "name": 1, "description": 1, "owner": 1, "ownerid": 1, "categoryid": 1, "category": 1, "content": 1, } }, ] res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] if res: data = list(res) return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) return JsonResponse({"code": 1001, "message": "no this method"}) # 文章模板详情 @csrf_exempt def getContent(request): _datecreated = timestamp_to_str() try: if request.method == "GET": contentid = request.GET.get('contentid', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] filter_ = {"_id": ObjectId(contentid)} params = { "_id": 0, "id": {f"$toString": f"$_id"}, "name": 1, "description": 1, "owner": 1, "ownerid": 1, "categoryid": 1, "category": 1, "content": 1, "wordgroups": 1, "property": 1, "datecreated": 1, "dateupdated": 1, } res = collection.find_one(filter_, params, no_cursor_timeout=True) return JsonResponse({ "code": "0000", "message": "成功", "data": res, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) return JsonResponse({"code": 1001, "message": "no this method"}) # 新增模板 @csrf_exempt def newContent(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) name = get_data.get('name', '') description = get_data.get('description', '') owner = get_data.get('owner', '') ownerid = get_data.get('ownerid', '') categoryid = get_data.get('categoryid', '') category = get_data.get('category', '') content = get_data.get('content', '') wordgroups = get_data.get('wordgroups', []) property = get_data.get('property', {"distinct": 3}) TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] data = { "name": name, "description": description, "owner": owner, "ownerid": ownerid, "categoryid": categoryid, "category": category, "content": content, "wordgroups": wordgroups, "property": property, "datecreated": TIME, "dateupdated": TIME, "sort": 100, "state": 0 } res = collection.insert_one(data) return JsonResponse({ "code": "0000", "message": "成功", "data": res.inserted_id, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) return JsonResponse({"code": 1001, "message": "no this method"}) # 修改模板 @csrf_exempt def updateContent(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) name = get_data.get('name', '') description = get_data.get('description', '') owner = get_data.get('owner', '') ownerid = get_data.get('ownerid', '') categoryid = get_data.get('categoryid', '') category = get_data.get('category', '') content = get_data.get('content', '') wordgroups = get_data.get('wordgroups', []) property = get_data.get('property', {"distinct": 3}) contentid = get_data.get('id', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] params = { "$set": { "name": name, "description": description, "owner": owner, "ownerid": ownerid, "categoryid": categoryid, "category": category, "content": content, "wordgroups": wordgroups, "property": property, "dateupdated": TIME, } } filter_ = {"_id": ObjectId(contentid)} res = collection.update_one(filter_, params) return JsonResponse({ "code": "0000", "message": "成功", "data": contentid, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) return JsonResponse({"code": 1001, "message": "no this method"}) # 删除模板 @csrf_exempt def deleteContent(request): _datecreated = timestamp_to_str() try: if request.method == "POST": get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) contentid = get_data.get('contentid', '') TIME = timestamp_to_str() with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_content] collection_del = db_[pongo_content + '_del'] filter_ = {"_id": ObjectId(contentid)} data = collection.find_one(filter_) data.update({"dateupdated": _datecreated}) collection_del.insert_one(data) res = collection.delete_one(filter_) return JsonResponse({ "code": "0000", "message": "成功", "data": contentid, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) return JsonResponse({"code": 1001, "message": "no this method"}) # 分类列表 @csrf_exempt def getCategoryList(request): _datecreated = timestamp_to_str() try: if request.method == 'POST': get_data = {} try: get_data = json.loads(request.body) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) return JsonResponse({"code": 0, "message": "success"}) if request.method == "GET": ownerid = request.GET.get('ownerid', '') owner = request.GET.get('owner', '') with MongoDBClient() as m_client: db_ = m_client[db] # 获取collection collection = db_[pongo_category] filter_ = {} if ownerid: filter_ = {"ownerid": ownerid} if owner: filter_ = {"owner": owner} pipeline = [ {"$match": filter_}, { "$project": { "_id": 0, "id": {f"$toString": f"$_id"}, "name": 1, "owner": 1, "ownerid": 1, } }, ] res = collection.aggregate(pipeline=pipeline, allowDiskUse=True) data = [] if res: data = list(res) return JsonResponse({ "code": "0000", "message": "成功", "data": data, "datecreated": _datecreated, }, json_dumps_params={'ensure_ascii': False, "default": json_util.default}) except Exception as e: return JsonResponse({"code": 1002, "message": str(e)}) return JsonResponse({"code": 1001, "message": "no this method"})
python爬虫的任务数据操作的小技巧
好久没写公众号了,最近太忙了,这里就不多说了。直接根据需求上代码,我想这个应该是大家比较喜欢的,
需求
爬取某网站的项目列表页,获取其url,标题等信息,作为后续爬取详情页的任务url。
先上代码
代码
# -*- coding: utf-8 -*-
# @Time : 2019-11-08 14:04
# @Author : cxa
# @File : motor_helper.py
# @Software: PyCharm
import asyncio
import datetime
from loguru import logger
from motor.motor_asyncio import AsyncIOMotorClient
from collections import Iterable
try:
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
except ImportError:
pass
db_configs = {
'host': '127.0.0.1',
'port': '27017',
'db_name': 'mafengwo',
'user': ''
}
class MotorOperation:
def __init__(self):
self.__dict__.update(**db_configs)
if self.user:
self.motor_uri = f"mongodb://{self.user}:{self.passwd}@{self.host}:{self.port}/{self.db_name}?authSource={self.db_name}"
else:
self.motor_uri = f"mongodb://{self.host}:{self.port}/{self.db_name}"
self.client = AsyncIOMotorClient(self.motor_uri)
self.mb = self.client[self.db_name]
async def save_data_with_status(self, items, col="seed_data"):
for item in items:
data = dict()
data["update_time"] = datetime.datetime.now()
data["status"] = 0 # 0初始
data.update(item)
print("data", data)
await self.mb[col].update_one({
"url": item.get("url")},
{'$set': data, '$setOnInsert': {'create_time': datetime.datetime.now()}},
upsert=True)
async def add_index(self, col="seed_data"):
# 添加索引
await self.mb[col].create_index('url')
因为我的爬虫是异步网络模块aiohttp写的,所以选择了pymongo的异步版本motor进行操作。
异步代码的基本属性就是async/await成对的出现,如果把上面的await和async去掉,就是类似pymongo的写法了,这里异步不是重点,重点是我们怎么处理每条数据。
这里除了网页的url,标题等信息,我需要附加3个字段。分别是create_time, status,update_time。
这三个字段分别代表,数据插入数据,状态和更新时间。
那么我为什么添加三个字段呢?
首先,我们需要判断每次的任务数据是否存在,我这里的情况是存在就更新不存在就插入,那么我就需要一个查询条件,作为更新的条件,很显然这里可以使用任务的url作为唯一条件(你还可以使用url+标题做个md5然后保存)。好了查询条件确定,
下面说create_time这个比较好理解就是数据插入时间,关键是为什么还要一个update_time,这个的话和status字段有一定的关系。 画重点:这个status作为后续爬虫进行爬取的一个标志用。目前这个status有4个值,0-4,我这是这样定义的,
0:初始状态 1:抓取中的任务 2:抓取成功 3:抓取失败 4:抓取成功但是没有匹配到任务。
后面随着任务的爬取,状态也是不断变化的,同时我们需要更新update_time为最新的时间。这个目前的话是体现不出来什么作用,它的使用场景是,重复任务的抓取,比如今天我抓取了任务列表里的url1,url2,第二天的时候我如果再抓到,为了区分是抓取失败还是抓取成功,我们根据create_time和update_time就可以进行推断了,如果两者相同而且是当前的日期说明刚抓的,如果update_time的日期比create_time新可以说明,抓到了重复的任务。关于字段的设计就啰嗦这么写。
下面是实现,我们可以通过update_one方法,对数据作存在或者插入操作,因为url作为查询条件,后面量大的话就最好添加一个索引。也就是上面的 add_index方法。
好了最好说插入更新的具体代码
需要注意的是
{'$set': data, '$setOnInsert': {'create_time': datetime.datetime.now()}}
$setOnInsert
里面使用的字段是数据不存在的时候才插入的,存在就不动了,只插入$set
里面指定的。
另外$setOnInsert
里面使用的字段不能在$set
里面再次出现
upsert=True代表的是不存在就插入。
大概就这么多,不明白的可以给我留言,或者添加微信进群交流。
[Ctrl+Enter快捷键提交]
【推荐】MASA Framework 开启全新的.NET应用开发体验
【推荐】下一步,敏捷!云可达科技SpecDD敏捷开发专区