正反向隔离装置

正反向隔离装置是做网络隔离,数据通过固定格式的文本进行传输,这里两台服务器之间的数据通过mqtt进行传输,所以使用脚本接受mqtt的json数据,然后转为固定格式的文本,在通过隔离装置传输与读取

 

main.py

复制代码
 1 #! /usr/bin/python
 2 # -*- coding: utf-8 -*-
 3 
 4 import threading
 5 
 6 import doSaveELanguage
 7 import doSendMessage
 8 
 9 if __name__ == '__main__':
10     sendThread = threading.Thread(target=doSendMessage.start_send, name='send_thread')
11     saveThread = threading.Thread(target=doSaveELanguage.start_save, name='save_thread')
12     sendThread.start()
13     saveThread.start()
复制代码

function.py  加载日志、config配置等公共信息

复制代码
 1 #! /usr/bin/python
 2 # -*- coding: utf-8 -*-
 3 import configparser
 4 import os
 5 import logging
 6 from logging import handlers
 7 import time
 8 
 9 
10 # 日志
11 class mediatorLog:
12     def __init__(self, __name__):
13         # 第一步，创建一个logger
14         self.logger = logging.getLogger(__name__)
15         self.logger.setLevel(logging.DEBUG)  # Log等级总开关
16 
17         # 第二步，创建一个handler，用于写入日志文件
18         logpath = '.\\log\\' + __name__ + '\\'
19         if (not os.path.exists(logpath)):
20             os.makedirs(logpath)
21         logName = 'log' + time.strftime('%Y%m%d%H%M') + '.txt'
22 
23         fh = handlers.TimedRotatingFileHandler(filename=logpath + logName, when='H', backupCount=15,
24                                                encoding='utf-8')
25         # fh = logging.FileHandler(logpath + logName, mode='a') # open的打开模式这里可以进行参考
26         fh.setLevel(logging.DEBUG)  # 输出到file的log等级的开关
27 
28         # 第三步，再创建一个handler，用于输出到控制台
29         ch = logging.StreamHandler()
30         ch.setLevel(logging.INFO)  # 输出到console的log等级的开关
31 
32         # 第四步，定义handler的输出格式
33         formatter = logging.Formatter(
34             "%(asctime)s - %(process)s - %(thread)s - %(filename)s[%(funcName)s(line:%(lineno)d)] - %(levelname)s: %(message)s")
35         fh.setFormatter(formatter)
36         ch.setFormatter(formatter)
37 
38         # 第五步，将logger添加到handler里面
39         self.logger.addHandler(fh)
40         self.logger.addHandler(ch)
41 
42         fh.close()
43         ch.close()
44 
45     def __getLog__(self):
46         return self.logger
47 
48 
49 # 配置文件
50 class configValue:
51     print('开始加载配置')
52     # 加载配置文件
53     cf = configparser.ConfigParser()
54     server_config = input('请输入需要加载的配置后缀: 当前配置有 150,201:')
55     config_name = 'config-'+server_config+'.ini'
56     print('当前加载配置文件名称为:'+config_name)
57     # 加载ini配置
58     cf.read(str(config_name), encoding="utf-8")
59     print('info.msg \t'+cf.get('configInfo','info.msg'))
60     print('info.ip \t' + cf.get('configInfo', 'info.ip'))
61 
62     mqttIp = cf.get('mqttConnect', 'mqtt.ip')
63     mqttPort = cf.get('mqttConnect', 'mqtt.port')
64     mqtt_send_topic = cf.get('mqttConnect', 'mqtt.sendTopic')
65     mqtt_send_topic_is_append = cf.get('mqttConnect', 'mqtt.sendTopic.isAppend')
66     mqtt_send_topic_is_append = bool(mqtt_send_topic_is_append)
67     mqtt_send_topic_append = cf.get('mqttConnect', 'mqtt.sendTopic.append')
68 
69     fileReadScan = cf.get('fileDir', 'fileDir.fileReadScan')
70     fileReadDir = cf.get('fileDir', 'fileDir.fileReadDir')
71     mqtt_save_topic = cf.get('mqttConnect', 'mqtt.saveTopic')
72     fileSaveScan = cf.get('fileDir', 'fileDir.fileSaveScan')
73     fileSaveDir = cf.get('fileDir', 'fileDir.fileSaveDir')
74 
75 
76 # 共享变量
77 class global_value(object):
78     # 发送端的日志
79     send_log = mediatorLog('send_thread_log').__getLog__()
80     # 接收端的日志
81     save_log = mediatorLog('save_thread_log').__getLog__()
82     # 保存时的全局变量
83     save_msg_list = list()
复制代码

doSendMessage.py 读取隔离装置传递的文本,并发送到Mqtt中

复制代码
#! /usr/bin/python
# -*- coding: utf-8 -*-
# 发送MQTT消息


import json
import os
import sys
import time
import uuid

import paho.mqtt.client as mqtt
from function import configValue, global_value


# UUID做为订阅ID
def getUUID():
    return "".join(str(uuid.uuid4()).split("-")).upper()


# 读取文件并且解析 固定格式
def read_file():
    global_value.send_log.info('======开始读取E文本格式数据======')
    new_fileReadDir = configValue.fileReadDir
    if not os.path.exists(new_fileReadDir):
        global_value.send_log.info(f'当前文件读取路径{new_fileReadDir}不存在,不进行解析')
        return list()
    file_name_list = os.listdir(new_fileReadDir)
    if len(file_name_list) == 0:
        global_value.send_log.info(f'当前文件读取路径{new_fileReadDir}下文件数量为0,不进行解析')
        return list()
    json_list = list()
    for file_name in file_name_list:
        global_value.send_log.info(f'======当前读取文件为:{file_name}======')
        file_data_list = list()
        try:
            with open(new_fileReadDir + file_name, 'r+', encoding='UTF-8') as f:
                # 读取
                heards = []
                line_index = 0
                for line in f:
                    line_index = line_index + 1
                    # 特殊行排除
                    if line.__contains__('TpCodeValue'):
                        continue
                    elif line_index == 2:
                        # 表头行
                        global_value.send_log.info(f'表头行内容为：{line}类型为{type(line)}')
                        heards = line.split('\t')[1].split(' ')
                    else:
                        # 数据行 排除第一位和最后一位
                        data_body = line.split('\t')[1:-1]
                        data_map = dict()
                        for i in range(len(heards)):
                            if data_body[i] == 'NoValue':
                                data_map[str(heards[i])] = ''
                            else:
                                data_map[str(heards[i])] = data_body[i]
                        file_data_list.append(data_map)
            json_list.extend(file_data_list)
            global_value.send_log.info(f'文件{file_name}解析出的数据量大小为{len(file_data_list)}')
            os.remove(new_fileReadDir+file_name)
        except Exception as e:
            global_value.send_log.error(f'解析文本 {file_name} 信息出现问题',e)
            continue
    print(f'一共读取数据量:{len(json_list)}条')
    return json_list


# 加载连接对象
def init_send_client() -> mqtt.Client:
    # 连接成功
    def on_connect(send_client, userdata, flags, rc):
        client_id = send_client._client_id.decode()
        if rc == 0:
            global_value.send_log.info(f'服务端 {client_id} 连接MQTT成功  连接状态:' + str(rc))
        else:
            global_value.send_log.info(f'服务端 {client_id} 连接MQTT失败  连接状态:' + str(rc))

    # 连接失败
    def on_disconnect(send_client, userdata, rc):
        client_id = send_client._client_id.decode()
        if rc != 0:
            global_value.save_log.info(f'服务端mqtt {client_id}连接失败,等待重新连接...')
            send_client.reconnect()
        else:
            global_value.save_log.info(f'服务端mqtt {client_id}重连成功')

    sendId = getUUID()
    client = mqtt.Client(sendId)
    client.username_pw_set("sendUser", "123456")
    client.on_connect = on_connect
    client.on_disconnect = on_disconnect
    client.connect(host=str(configValue.mqttIp), port=int(configValue.mqttPort), keepalive=600)
    client.loop_start()
    global_value.send_log.info('加载mqtt服务端:' + str(sendId))
    return client


# 发送消息
def send_msg(client: mqtt.Client):
    client_id = client._client_id.decode()
    while True:
        list = read_file()
        if len(list) > 0:
            send_count = 0
            for data in list:
                send_count = send_count + 1
                sendTopic = configValue.mqtt_send_topic
                # 如果开启自动拼接
                if configValue.mqtt_send_topic_is_append:
                    # 存在拼接的字段
                    if data.keys().__contains__(configValue.mqtt_send_topic_append):
                        if data[configValue.mqtt_send_topic_append] == '' or data[
                            configValue.mqtt_send_topic_append] is None:
                            sendTopic = configValue.mqtt_send_topic.replace(
                                '${' + configValue.mqtt_send_topic_append + '}', '1')
                        else:
                            sendTopic = configValue.mqtt_send_topic.replace(
                                '${' + configValue.mqtt_send_topic_append + '}',
                                data[configValue.mqtt_send_topic_append])
                            # sendTopic = configValue.mqtt_send_topic + data[configValue.mqtt_send_topic_append]
                send_value = json.dumps(data)
                result = client.publish(topic=sendTopic, payload=send_value, qos=0)
                if result[0] == 0:
                    global_value.send_log.info(f'发送端{client_id} 向队列 {sendTopic} 发送第{send_count}次消息 {send_value} 成功')
                else:
                    global_value.send_log.info(f'发送端{client_id} 向队列 {sendTopic} 发送第{send_count}次消息 {send_value} 失败')
        else:
            global_value.send_log.info('当前无数据需要发送')
        time.sleep(int(configValue.fileReadScan))


def start_send():
    global_value.send_log.info('======开始加载MQTT服务端连接=====')
    sendClient = init_send_client()
    try:
        global_value.send_log.info('======开始发送消息======')
        send_msg(sendClient)
    except KeyboardInterrupt:
        global_value.send_log.error('======MQTT服务端关闭======')
        sendClient.disconnect()
        sys.exit(0)


if __name__ == '__main__':
    start_send()
复制代码

doSaveELanguage.py 接收mqtt传递的数据,并进行固定文本格式的存储

复制代码
#! /usr/bin/python
# -*- coding: utf-8 -*-
# 接收mqtt消息,并且存储为e语言格式文本

import os
import sys
import time
import uuid
import json

import paho.mqtt.client as mqtt
import threading

from function import configValue, global_value


# UUID做为订阅ID
def getUUID():
    return "".join(str(uuid.uuid4()).split("-")).upper()


# 存储数据到文件中 固定格式 @data_list 为json集合
def write_file(data_list: list):
    global_value.save_log.info('======开始存储数据为E文本格式======')
    # 获取表头
    header = ['@顺序', tuple([i for i in data_list[0].keys()])]
    val_list = []
    for row_index in range(len(data_list)):
        data = data_list[row_index]
        t_body = ['#' + str(row_index + 1)]
        for v in data.values():
            if v is None or v == 'Null' or v == '':
                t_body.append('NoValue')
            else:
                t_body.append(v)
        val_list.append(tuple(t_body))
    # 写入
    now_date = time.strftime('%Y%m%d')
    # 判断路径已经文件是否存在,创建文件夹
    new_fileSaveDir = configValue.fileSaveDir
    if not os.path.exists(new_fileSaveDir):
        os.makedirs(new_fileSaveDir)
    # 存在上次数据未处理完成的状况
    file_name_list = os.listdir(new_fileSaveDir)
    # 文件真实名称
    file_abs_name = ''
    if len(file_name_list) == 0:
        file_abs_name = now_date + '.txt'
    else:
        file_abs_name = now_date + '_' + str(len(file_name_list)) + '.txt'
    txtVal = ''
    fw = open(new_fileSaveDir + file_abs_name, 'w+', encoding='utf-8')
    now_date = time.strftime('%Y-%m-%d %H:%M:%S')
    txtVal += f'<TpCodeValue date=\'{now_date}\'>\n'
    # 表头
    for h in header:
        s = str(h).replace('[', '').replace(']', '').replace('(', '').replace(')', '')
        s = s.replace("'", '').replace(',', '') + '\t'
        txtVal+=s
    txtVal+='\n'
    for v in val_list:
        for d in v:
            s = str(d).replace('[', '').replace(']', '').replace('(', '').replace(')', '')
            s = s.replace("'", '').replace(',', '') + '\t'
            txtVal+=s
        txtVal+='\n'
    txtVal+='</TpCodeValue>'
    fw.write(txtVal)
    fw.flush()
    fw.close()
    global_value.save_log.info(f'======结束存储数据为E文本格式,文件名为{file_abs_name}======')


# 加载连接对象
def init_save_client() -> mqtt.Client:
    # 连接成功
    def on_connect(save_client, userdata, flags, rc):
        client_id = save_client._client_id.decode()
        if rc == 0:
            global_value.save_log.info(f'客户端 {client_id} 连接MQTT成功  连接状态:' + str(rc))
        else:
            global_value.save_log.info(f'客户端 {client_id} 连接MQTT失败  连接状态:' + str(rc))

    # 连接失败
    def on_disconnect(save_client, userdata, rc):
        client_id = save_client._client_id.decode()
        if rc != 0:
            global_value.save_log.info(f'客户端mqtt {client_id}连接失败,等待重新连接...')
            save_client.reconnect()
        else:
            global_value.save_log.info(f'客户端mqtt {client_id}重连成功')

    # 接收消息
    def on_message(save_client, userdata, msg):
        msgPayload = msg.payload.decode()
        msgTopic = msg.topic
        client_id = save_client._client_id.decode()
        # 存储到内部缓存中
        global_value.save_msg_list.append(json.loads(msgPayload))
        global_value.save_log.info(f'客户端 {client_id} 从Topic{msgTopic} 获取到数据 {msgPayload}')

    resolveId = getUUID()
    client = mqtt.Client(resolveId)
    client.username_pw_set("resolveUser", "123456")
    client.on_connect = on_connect
    client.on_disconnect = on_disconnect
    client.connect(host=str(configValue.mqttIp), port=int(configValue.mqttPort), keepalive=600)
    global_value.save_log.info('加载mqtt客户端:' + str(resolveId))
    client.subscribe(configValue.mqtt_save_topic)
    client.on_message = on_message
    client.loop_forever()
    return client


def start_save():
    global_value.save_log.info('======开始加载MQTT客户端连接=====')
    save_thread = threading.Thread(target=init_save_client, name='save_thread')
    save_thread.start()
    try:
        # 固定时间扫描一次
        save_count = 0
        while True:
            save_count = save_count + 1
            global_value.save_log.info(f'当前为第{save_count}次扫描')
            # 转移缓存
            now_list = list()
            now_list.extend(global_value.save_msg_list)
            global_value.save_msg_list = list()
            if len(now_list) > 0:
                global_value.save_log.info(f'本次存储数据数量为:{len(now_list)}')
                try:
                    write_thread = threading.Thread(target=write_file, name='write_file_thread_' + str(save_count),
                                                    args=[now_list])
                    write_thread.start()
                except Exception as e:
                    global_value.save_log.error('文件写入失败',e)
                    continue
            else:
                global_value.save_log.info('当前mqtt客户端未接收到数据,不进行存储')
            time.sleep(int(configValue.fileSaveScan))
    except KeyboardInterrupt:
        global_value.save_log.error("======MQTT客户端关闭======")
        sys.exit(0)

# if __name__ == '__main__':
#     start_save()

if __name__ == '__main__':
    # read_file()
    obj = json.load(open('12312.json'))
    data_list = obj['RECORDS']
    write_file(data_list)
复制代码

导包 pip install paho-mqtt

配置文件1 config-150.ini

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
;配置文件说明
[configInfo]
info.msg = 部署于172.30.50.150服务中
info.ip = 172.30.50.150
;mqtt连接配置
[mqttConnect]
;mqtt连接Ip
mqtt.ip = 192.168.99.101
;mqtt连接端口
mqtt.port = 32779
;发送消息的队列名称
mqtt.sendTopic = /V1/${server_code}/device/data/reply
;队列是是否动态自动拼接
mqtt.sendTopic.isAppend = True
;拼接数据字段
mqtt.sendTopic.append = server_code
;接收消息的队列名称
mqtt.saveTopic = /V1/+/device/data
 
;隔离装置配置的文件路径
[fileDir]
;存储路径
fileDir.fileSaveDir = ./home/ftzxjc/geli/send/fs/
;存储频率:秒
fileDir.fileSaveScan = 5
;读取路径
fileDir.fileReadDir = ./home/ftzxjc/geli/send/fs/
;读取频率
fileDir.fileReadScan = 15

　配置文件2 config-201.ini

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
;配置文件说明
[configInfo]
info.msg = 部署于172.30.51.201服务中
info.ip = 172.30.51.201
;mqtt连接配置
[mqttConnect]
;mqtt连接Ip
mqtt.ip = 192.168.99.101
;mqtt连接端口
mqtt.port = 32779
;发送消息的队列名称
mqtt.sendTopic = /V1/${server_code}/device/data
;队列是是否动态自动拼接
mqtt.sendTopic.isAppend = True
;拼接数据字段
mqtt.sendTopic.append = server_code
;接收消息的队列名称
mqtt.saveTopic = /V1/+/device/data/reply
 
;隔离装置配置的文件路径
[fileDir]
;存储路径
fileDir.fileSaveDir = D:/python_workspace/elanguageProject/home/user/send/fs/
;存储频率:秒
fileDir.fileSaveScan = 5
;读取路径
fileDir.fileReadDir = D:/python_workspace/elanguageProject/home/user/recv/js/isolate_recv/
;读取频率
fileDir.fileReadScan = 15

　　

测试文件

12312.json

 

　　

posted @   雨梦大木  阅读(1477)  评论(0)  编辑  收藏  举报
相关博文：
·  websocket2.0 适用于发送的数据体很大
·  druid数据源手动创建
·  用pandas读取csv文件，并删除包含NaN的数值行，计算df列的length
·  java接收数组并解析
·  coding
阅读排行：
· 震惊！C++程序真的从main开始吗？99%的程序员都答错了
· 别再用vector＜bool＞了！Google高级工程师：这可能是STL最大的设计失误
· 单元测试从入门到精通
· 【硬核科普】Trae如何「偷看」你的代码？零基础破解AI编程运行原理
· 上周热点回顾（3.3-3.9）
点击右上角即可分享
微信分享提示
AI IDE Trae