以下代码用于在swin transformer目标检测官方代码训练过程中生成的json文件中获取验证集每个epoch的bbox_mAP_50，代码如下：
import json
import matplotlib.pyplot as plt
import argparse
import numpy as np
'''
解析参数
'''
parser = argparse.ArgumentParser()
parser.add_argument("--mode", type=str, default='val')
parser.add_argument("--select", type=str, default='bbox_mAP_50')
parser.add_argument("--json_paths", type=str, nargs='+')
parser.add_argument("--line_names", type=str, nargs='+')
parser.add_argument("--out_dir", type=str, default='./')
parser.add_argument("--epoch_num", type=int, default=70)
parser.add_argument("--pic_name", type=str,default="result")
args = parser.parse_args()
select=args.select
pic_name=args.pic_name
mode = args.mode # 选择log文件中的模式
json_paths = args.json_paths
line_names = args.line_names
out_dir = args.out_dir
epoch_num = args.epoch_num
plt.figure(figsize=(12, 8), dpi=300)
for i, json_path in enumerate(json_paths):
epoch_now = 0
x = [] # 存放epoch
y = [] # 存放指标
y_min = 1000000 # 存放指标最大值 ap不会超过1 绘制loss可自由更改
y_max = -1 # 存放指标最小值 ap不会小于-1 绘制loss可自由更改
x_min = 0 # 出现最小值的epoch
x_max = 0 # 出现最大值的epoch
isFirst = True
with open(json_path, 'r') as f:
for jsonstr in f.readlines():
if epoch_now == epoch_num:
break
if isFirst: # mmdetection生成的log json文件第一行是配置信息 跳过
isFirst = False
continue
row_data = json.loads(jsonstr)
if row_data['mode'] == mode: # 选择train或者val模式中的指标数据
epoch_now = epoch_now + 1
item_select = float(row_data[select])
x_select = int(row_data['epoch'])#enpoch
x.append(x_select)
y.append(item_select)
if item_select >= y_max: # 选择最大值 为什么不用numpy.argmin呢？ 因为epoch可能不从1开始 xmin和ymin可能匹配错误 比较麻烦
y_max = item_select
x_max = x_select
if item_select <= y_min: # 选择最大值
y_min = item_select
x_min = x_select
plt.grid(True, linestyle='--', alpha=0.5)
plt.plot(x, y, label=line_names[i])
plt.plot(x_min, y_min, 'g-p', x_max, y_max, 'r-p')
show_min = '[' + str(x_min) + ' , ' + str(y_min) + ']'
show_max = '[' + str(x_max) + ' , ' + str(y_max) + ']'
plt.annotate(show_min, xy=(x_min, y_min), xytext=(x_min, y_min))
plt.annotate(show_max, xy=(x_max, y_max), xytext=(x_max, y_max))
plt.xlabel('epoch')
plt.legend()
plt.ylabel(select)
# plt.ylim(0.8, 1.0) # 设置y轴坐标范围
plt.savefig(args.out_dir + '/' + pic_name + '.jpg', dpi=300)
但是生成的json文件是按iter计算loss的，因此一个epoch就有多个loss导致以上代码不能对json文件绘制出横轴为epoch 纵轴为loss的损失函数曲线。生成的json文件内格式大致如下（第一行一般记录硬件信息）：
{"mode": "train", "epoch": 1, "iter": 50, "lr": 0.0, "memory": 2457, "data_time": 0.08238, "loss_rpn_cls": 0.11479, "loss_rpn_bbox": 0.01201, "loss_cls": 6.32701, "acc": 16.02637, "loss_bbox": 0.09524, "loss": 6.54904, "time": 0.46118}
{"mode": "train", "epoch": 1, "iter": 100, "lr": 0.0, "memory": 2458, "data_time": 0.01115, "loss_rpn_cls": 0.10915, "loss_rpn_bbox": 0.01282, "loss_cls": 3.66955, "acc": 46.08594, "loss_bbox": 0.09219, "loss": 3.88372, "time": 0.40142}
{"mode": "train", "epoch": 1, "iter": 150, "lr": 0.0, "memory": 2458, "data_time": 0.01097, "loss_rpn_cls": 0.09242, "loss_rpn_bbox": 0.01081, "loss_cls": 1.3539, "acc": 88.45068, "loss_bbox": 0.10293, "loss": 1.56006, "time": 0.39346}
{"mode": "train", "epoch": 1, "iter": 200, "lr": 0.0, "memory": 2458, "data_time": 0.01132, "loss_rpn_cls": 0.08857, "loss_rpn_bbox": 0.01041, "loss_cls": 0.68504, "acc": 94.40088, "loss_bbox": 0.11103, "loss": 0.89505, "time": 0.40342}
{"mode": "train", "epoch": 1, "iter": 250, "lr": 0.0, "memory": 2458, "data_time": 0.01095, "loss_rpn_cls": 0.08148, "loss_rpn_bbox": 0.01066, "loss_cls": 0.50034, "acc": 95.19434, "loss_bbox": 0.10805, "loss": 0.70053, "time": 0.38798}
{"mode": "train", "epoch": 1, "iter": 300, "lr": 1e-05, "memory": 2458, "data_time": 0.01136, "loss_rpn_cls": 0.06545, "loss_rpn_bbox": 0.01087, "loss_cls": 0.41874, "acc": 95.13135, "loss_bbox": 0.12165, "loss": 0.61671, "time": 0.39193}
{"mode": "train", "epoch": 1, "iter": 350, "lr": 1e-05, "memory": 2458, "data_time": 0.01126, "loss_rpn_cls": 0.05331, "loss_rpn_bbox": 0.00946, "loss_cls": 0.36316, "acc": 95.54492, "loss_bbox": 0.11369, "loss": 0.53962, "time": 0.39703}
{"mode": "train", "epoch": 1, "iter": 400, "lr": 1e-05, "memory": 2458, "data_time": 0.01131, "loss_rpn_cls": 0.04951, "loss_rpn_bbox": 0.00969, "loss_cls": 0.36704, "acc": 95.26758, "loss_bbox": 0.11932, "loss": 0.54555, "time": 0.39744}
{"mode": "train", "epoch": 1, "iter": 450, "lr": 1e-05, "memory": 2458, "data_time": 0.01114, "loss_rpn_cls": 0.0408, "loss_rpn_bbox": 0.00939, "loss_cls": 0.31964, "acc": 95.36572, "loss_bbox": 0.1184, "loss": 0.48823, "time": 0.39942}
{"mode": "train", "epoch": 1, "iter": 500, "lr": 1e-05, "memory": 2458, "data_time": 0.01115, "loss_rpn_cls": 0.03528, "loss_rpn_bbox": 0.00916, "loss_cls": 0.30343, "acc": 95.22168, "loss_bbox": 0.12234, "loss": 0.47021, "time": 0.39602}
{"mode": "train", "epoch": 1, "iter": 550, "lr": 1e-05, "memory": 2458, "data_time": 0.01108, "loss_rpn_cls": 0.03108, "loss_rpn_bbox": 0.00969, "loss_cls": 0.29804, "acc": 95.02979, "loss_bbox": 0.12529, "loss": 0.4641, "time": 0.39857}
{"mode": "val", "epoch": 1, "iter": 591, "lr": 1e-05, "bbox_mAP": 0.042, "bbox_mAP_50": 0.096, "bbox_mAP_75": 0.029, "bbox_mAP_s": 0.002, "bbox_mAP_m": 0.114, "bbox_mAP_l": 0.254, "bbox_mAP_copypaste": "0.042 0.096 0.029 0.002 0.114 0.254"}
{"mode": "train", "epoch": 2, "iter": 50, "lr": 1e-05, "memory": 2458, "data_time": 0.08325, "loss_rpn_cls": 0.02446, "loss_rpn_bbox": 0.00831, "loss_cls": 0.29634, "acc": 95.13184, "loss_bbox": 0.12194, "loss": 0.45105, "time": 0.46476}
以上代码为json文件中的一部分
请问如何修改代码才能从json文件中获取每个epoch的loss并绘制损失曲线
引用chatGPT作答，要从json文件中获取每个epoch的loss并绘制损失函数曲线，需要对代码进行一些修改。具体来说，需要将x和y存放的数据改为epoch的loss值，而不是指标值（如bbox_mAP_50）。此外，需要在每个epoch的最后一行绘制该epoch的平均损失函数值。修改后的代码如下：
import json
import matplotlib.pyplot as plt
import argparse
import numpy as np
'''
解析参数
'''
parser = argparse.ArgumentParser()
parser.add_argument("--mode", type=str, default='val')
parser.add_argument("--select", type=str, default='loss') # 修改为loss
parser.add_argument("--json_paths", type=str, nargs='+')
parser.add_argument("--line_names", type=str, nargs='+')
parser.add_argument("--out_dir", type=str, default='./')
parser.add_argument("--epoch_num", type=int, default=70)
parser.add_argument("--pic_name", type=str,default="result")
args = parser.parse_args()
select=args.select
pic_name=args.pic_name
mode = args.mode # 选择log文件中的模式
json_paths = args.json_paths
line_names = args.line_names
out_dir = args.out_dir
epoch_num = args.epoch_num
plt.figure(figsize=(12, 8), dpi=300)
for i, json_path in enumerate(json_paths):
epoch_now = 0
x = [] # 存放epoch
y = [] # 存放loss
y_min = 1000000 # 存放指标最大值 ap不会超过1 绘制loss可自由更改
y_max = -1 # 存放指标最小值 ap不会小于-1 绘制loss可自由更改
x_min = 0 # 出现最小值的epoch
x_max = 0 # 出现最大值的epoch
isFirst = True
sum_loss = 0.0 # 记录当前epoch的总损失
iter_count = 0 # 记录当前epoch的迭代次数
with open(json_path, 'r') as f:
for jsonstr in f.readlines():
if epoch_now == epoch_num:
break
if isFirst: # mmdetection生成的log json文件第一行是配置信息 跳过
isFirst = False
continue
row_data = json.loads(jsonstr)
if row_data['mode'] == mode: # 选择train或者val模式中的指标数据
iter_count += 1 # 计算当前epoch的迭代次数
sum_loss += row_data['loss'] # 计算当前epoch的总损失
if row_data['epoch'] > epoch_now: # 到达下一个epoch
epoch_now = row_data['epoch']
x.append(epoch_now)
y.append(sum_loss / iter_count) # 将平均损失添加到y列表中
sum_loss = 0.0
iter_count = 0
plt.grid(True, linestyle='--', alpha=0.5)
plt.plot(x, y, label=line_names[i])
show_min = '[' + str(x_min) + ' , ' + str(y_min) + ']'
show_max = '[' + str(x_max) + ' , ' + str(y_max) + ']'
plt.annotate(show_min, xy=(x_min, y_min), xytext=(x_min, y_min + 0.5), color='r')
plt.annotate(show_max, xy=(x_max, y_max), xytext=(x_max, y_max - 0.5), color='show_max = '[' + str(x_max) + ' , ' + str(y_max) + ']'
if select == 'loss':
plt.title("Loss Graph")
plt.ylabel('Loss')
else:
plt.title("AP Graph")
plt.ylabel('AP')
plt.xlabel('Epoch')
plt.legend(loc='best')
plt.annotate('min ' + show_min, xy=(x_min, y_min), xytext=(x_min, y_min+0.1),
arrowprops=dict(facecolor='red', shrink=0.01),
)
plt.annotate('max ' + show_max, xy=(x_max, y_max), xytext=(x_max, y_max-0.1),
arrowprops=dict(facecolor='red', shrink=0.01),
)
plt.savefig(out_dir + pic_name + '_{}.png'.format(line_names[i]), dpi=300)
plt.show()
以下答案由GPT-3.5大模型与博主波罗歌共同编写：
首先需要对代码进行一些修改，以便从json文件中获取每个epoch的loss。代码修改如下：
import json
import matplotlib.pyplot as plt
import argparse
import numpy as np
parser = argparse.ArgumentParser()
parser.add_argument("--json_path", type=str, default='./log.json')
parser.add_argument("--mode", type=str, default='val')
parser.add_argument("--out_dir", type=str, default='./')
parser.add_argument("--pic_name", type=str,default="result")
args = parser.parse_args()
mode = args.mode
json_path = args.json_path
out_dir = args.out_dir
plt.figure(figsize=(12, 8), dpi=300)
x = [] # 存放epoch
y = [] # 存放loss
with open(json_path, 'r') as f:
current_epoch = 0
current_loss = 0.0
for jsonstr in f.readlines():
row_data = json.loads(jsonstr)
if row_data['mode'] == mode: # 选择train或者val模式中的指标数据
epoch = int(row_data['epoch'])
if epoch != current_epoch:
x.append(current_epoch)
y.append(current_loss)
current_loss = 0.0
current_epoch = epoch
loss = float(row_data.get('loss', 0.0))
current_loss += loss
x.append(current_epoch)
y.append(current_loss)
plt.grid(True, linestyle='--', alpha=0.5)
plt.plot(x, y, label='loss')
plt.xlabel('epoch')
plt.ylabel('loss')
plt.legend()
plt.savefig(out_dir + '/' + args.pic_name + '.jpg', dpi=300)
修改后的代码使用--json_path参数指定json文件路径。代码将遍历json文件中的所有行，对于每个epoch，将对应的loss相加，并将epoch和相加后的loss添加到x和y列表中。最后绘制loss曲线。
请注意，由于json文件中的每个行代表一个iteration，因此需要在代码中进行处理，将一个epoch中的所有iteration的loss相加，得到该epoch的loss。在上述代码中，我们通过记录当前的epoch和一个epoch的累计loss来实现此目的。
如果我的回答解决了您的问题，请采纳！