# coding:utf-8
# make_msg_dict.py
line_num = 0
new_block_flag = False
second_flag = False
linux_resource_flag = False
next_predict_flag = False
second_line = ""
threads_num = 0
process_num = 0
torch_threads_num = 0
img_num = 0
batch_size = 0
msg_line_num = 0
linux_resource = 0
all_count = 0
min_list = []
avg_list = []
max_list = []
fps_list = []
all_dict = {}
msg_dict = {}
other_count = 0
gpu = 0
gpu_memory = 0
cpu = 0
cpu_memory = 0
log_file = "rt_t2_process_12-10_19-00.log"
with open(log_file, 'r')as f:
for i in f:
line_num += 1
# 回车行
if i == "\n":
new_block_flag = False
continue
try:
# 每行内容
line_msg = i.split("INFO")[1].strip()
except Exception as error:
continue
# INFO 空行
if not line_msg:
continue
# 第一行
if line_msg == "predict_start":
second_flag = True
next_predict_flag = False
msg_dict = {}
min_list = []
avg_list = []
max_list = []
fps_list = []
gpu = 0
gpu_memory = 0
cpu = 0
cpu_memory = 0
continue
# 取第二行
if second_flag:
second_line = line_msg
second_ret_list = second_line.replace(" ", '').split(",")
# print(second_ret_list)
judge_type = second_ret_list[0].split(":")[0]
if judge_type == "process_num":
threads_num = -1
process_num = int(second_ret_list[0].split(":")[1])
msg_line_num = process_num * 5
else:
process_num = -1
threads_num = int(second_ret_list[0].split(":")[1])
msg_line_num = threads_num * 5
torch_threads_num = int(second_ret_list[1].split(":")[1])
img_num = int(second_ret_list[2].split(":")[1])
batch_size = int(second_ret_list[3].split(":")[1])
second_flag = False
continue
# 取 predict_msg
if msg_line_num > 0:
msg_line_num -= 1
if msg_line_num == 0:
linux_resource_flag = True
linux_resource = 17
# predict time
if ":" in line_msg:
line_msg_list = line_msg.replace(" ", '').split(":")
min_list.append(float(line_msg_list[1].split("/")[0]))
avg_list.append(float(line_msg_list[1].split("/")[1]))
max_list.append(float(line_msg_list[1].split("/")[2]))
continue
if "=" in line_msg:
line_msg_list = line_msg.replace(" ", '').split("=")
if line_msg_list[0][-3:] == "fps":
fps_list.append(float(line_msg_list[1]))
continue
continue
# 拿到linux数据
if linux_resource > 0:
linux_resource -= 1
if linux_resource == 0:
next_predict_flag = True
linux_resource_flag = False
# print(line_msg)
line_msg.strip()
if line_msg.startswith("cpu utilization"):
if line_msg.count("/") == 1:
cpu = str(int(float(line_msg.split(" ")[2][:-1]))) + "%"
continue
if line_msg.startswith("memory"):
if line_msg.count("/") == 1:
cpu_memory = str(round(float(line_msg.split(" ")[1][:-2]), 2) * 1000) + "MB"
continue
if line_msg.startswith("GPU 0 memory_used"):
gpu_memory = str(round(float(line_msg.split(":")[1].split("/")[1].replace(" ", '')[:-1]), 2)) + "MB"
continue
if line_msg.startswith("GPU 0 Utilization_Rates"):
gpu = str(int(float(line_msg.split(":")[1].split("/")[1].replace(" ", '')[:-1]))) + "%"
continue
# 取predict下的东西
if not linux_resource_flag and next_predict_flag:
# print("计数")
other_count += 1
msg_dict = {
"Throughput": round(sum(fps_list), 0) * batch_size,
"Latency": "{} / {} / {}".format(
round(min(min_list) * 1000, 2),
round(sum(avg_list) / len(avg_list) * 1000, 2),
round(max(max_list) * 1000, 2),
),
"gpu": gpu,
"gpu_memory": gpu_memory,
"cpu": cpu,
"cpu_memory": cpu_memory,
}
# 组合dict
if threads_num != -1:
if all_dict.get(1):
if all_dict[1].get(threads_num):
if all_dict[1][threads_num].get(batch_size):
all_dict[1][threads_num][batch_size][torch_threads_num] = msg_dict
else:
all_dict[1][threads_num][batch_size] = {torch_threads_num: msg_dict}
else:
all_dict[1][threads_num] = {batch_size: {torch_threads_num: msg_dict}}
else:
all_dict[1] = {threads_num: {batch_size: {torch_threads_num: msg_dict}}}
else:
if all_dict.get(process_num):
if all_dict[process_num].get(1):
if all_dict[process_num][1].get(batch_size):
all_dict[process_num][1][batch_size][torch_threads_num] = msg_dict
else:
all_dict[process_num][1][batch_size] = {torch_threads_num: msg_dict}
else:
all_dict[process_num][1] = {batch_size: {torch_threads_num: msg_dict}}
else:
all_dict[process_num] = {1: {batch_size: {torch_threads_num: msg_dict}}}
print("other_count", other_count)
print("all_dict", all_dict)
# make csv
# coding:utf-8
# make_csv.py
line_num = 0
new_block_flag = False
second_flag = False
linux_resource_flag = False
next_predict_flag = False
second_line = ""
threads_num = 0
process_num = 0
torch_threads_num = 0
img_num = 0
batch_size = 0
msg_line_num = 0
linux_resource = 0
all_count = 0
min_list = []
avg_list = []
max_list = []
fps_list = []
all_dict = {}
msg_dict = {}
other_count = 0
gpu = 0
gpu_memory = 0
cpu = 0
cpu_memory = 0
import os
log_dir_list = os.listdir("log_dir")
for log_file in log_dir_list:
all_dict = {}
other_count = 0
with open("log_dir/{}".format(log_file), 'r')as f:
for i in f:
line_num += 1
# 回车行
if i == "\n":
new_block_flag = False
continue
try:
# 每行内容
line_msg = i.split("INFO")[1].strip()
except Exception as error:
continue
# INFO 空行
if not line_msg:
continue
# 第一行
if line_msg == "predict_start":
second_flag = True
next_predict_flag = False
msg_dict = {}
min_list = []
avg_list = []
max_list = []
fps_list = []
gpu = 0
gpu_memory = 0
cpu = 0
cpu_memory = 0
continue
# 取第二行
if second_flag:
second_line = line_msg
second_ret_list = second_line.replace(" ", '').split(",")
# print(second_ret_list)
judge_type = second_ret_list[0].split(":")[0]
if judge_type == "process_num":
threads_num = -1
process_num = int(second_ret_list[0].split(":")[1])
msg_line_num = process_num * 5
else:
process_num = -1
threads_num = int(second_ret_list[0].split(":")[1])
msg_line_num = threads_num * 5
torch_threads_num = int(second_ret_list[1].split(":")[1])
img_num = int(second_ret_list[2].split(":")[1])
batch_size = int(second_ret_list[3].split(":")[1])
second_flag = False
continue
# 取 predict_msg
if msg_line_num > 0:
msg_line_num -= 1
if msg_line_num == 0:
linux_resource_flag = True
linux_resource = 17
# predict time
if ":" in line_msg:
line_msg_list = line_msg.replace(" ", '').split(":")
min_list.append(float(line_msg_list[1].split("/")[0]))
avg_list.append(float(line_msg_list[1].split("/")[1]))
max_list.append(float(line_msg_list[1].split("/")[2]))
continue
if "=" in line_msg:
line_msg_list = line_msg.replace(" ", '').split("=")
if line_msg_list[0][-3:] == "fps":
fps_list.append(float(line_msg_list[1]))
continue
continue
# 拿到linux数据
if linux_resource > 0:
linux_resource -= 1
if linux_resource == 0:
next_predict_flag = True
linux_resource_flag = False
# print(line_msg)
line_msg.strip()
if line_msg.startswith("cpu utilization"):
if line_msg.count("/") == 1:
cpu = str(int(float(line_msg.split(" ")[2][:-1]))) + "%"
continue
if line_msg.startswith("memory"):
if line_msg.count("/") == 1:
cpu_memory = str(round(float(line_msg.split(" ")[1][:-2]), 2) * 1000) + "MB"
continue
if line_msg.startswith("GPU 0 memory_used"):
gpu_memory = str(round(float(line_msg.split(":")[1].split("/")[1].replace(" ", '')[:-1]), 2)) + "MB"
continue
if line_msg.startswith("GPU 0 Utilization_Rates"):
gpu = str(int(float(line_msg.split(":")[1].split("/")[1].replace(" ", '')[:-1]))) + "%"
continue
# 取predict下的东西
if not linux_resource_flag and next_predict_flag:
# print("计数")
other_count += 1
msg_dict = {
"Throughput": round(sum(fps_list), 0) * batch_size,
"Latency": "{} / {} / {}".format(
round(min(min_list) * 1000, 2),
round(sum(avg_list) / len(avg_list) * 1000, 2),
round(max(max_list) * 1000, 2),
),
"gpu": gpu,
"gpu_memory": gpu_memory,
"cpu": cpu,
"cpu_memory": cpu_memory,
}
# 组合dict
if threads_num != -1:
if all_dict.get(1):
if all_dict[1].get(threads_num):
if all_dict[1][threads_num].get(batch_size):
all_dict[1][threads_num][batch_size][torch_threads_num] = msg_dict
else:
all_dict[1][threads_num][batch_size] = {torch_threads_num: msg_dict}
else:
all_dict[1][threads_num] = {batch_size: {torch_threads_num: msg_dict}}
else:
all_dict[1] = {threads_num: {batch_size: {torch_threads_num: msg_dict}}}
else:
if all_dict.get(process_num):
if all_dict[process_num].get(1):
if all_dict[process_num][1].get(batch_size):
all_dict[process_num][1][batch_size][torch_threads_num] = msg_dict
else:
all_dict[process_num][1][batch_size] = {torch_threads_num: msg_dict}
else:
all_dict[process_num][1] = {batch_size: {torch_threads_num: msg_dict}}
else:
all_dict[process_num] = {1: {batch_size: {torch_threads_num: msg_dict}}}
print("other_count", other_count)
# print("all_dict", all_dict)
# make csv
cvs_str = ''
all_cvs_str = ''
threads_count = 0
for process, threads_num_dict in all_dict.items():
cvs_str = str(process) + ","
for threads_num, batch_size_dict in threads_num_dict.items():
th_before_cvs_str = cvs_str
cvs_str += str(threads_num) + ","
for batch_size, torch_threads_num_dict in batch_size_dict.items():
b_before_cvs_str = cvs_str
cvs_str += str(batch_size) + ","
for torch_threads_num, msg_dict in torch_threads_num_dict.items():
t_before_cvs_str = cvs_str
cvs_str += str(torch_threads_num) + ","
msg_str = ",".join([str(i) for i in list(msg_dict.values())])
cvs_str += str(msg_str)
all_cvs_str += cvs_str + "\n"
cvs_str = t_before_cvs_str
cvs_str = b_before_cvs_str
cvs_str = th_before_cvs_str
print(all_cvs_str.count("\n"))
with open("{}.csv".format(log_file), 'w') as f:
f.write(all_cvs_str)
all_cvs_str = ""
