使用IBM ART库生成交通信号牌的攻击样本

目标：生成对抗样本，扰动图像，让原本是“停”的信号牌识别为“禁止驶入”：

代码如下（注意，因为我找的cnn原始模型支持的是灰度图像，所以彩色的对抗样本还需要修改代码）：

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

import cv2, os
import numpy as np
import numpy as np
import tensorflow as tf
from keras.models import load_model
from tensorflow.keras.layers import Dense
from tensorflow.keras.utils import to_categorical
from art.estimators.classification import TensorFlowV2Classifier
from art.attacks.evasion import FastGradientMethod
import matplotlib.pyplot as plt
from tensorflow.keras.models import load_model
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.utils import to_categorical
from sklearn.model_selection import train_test_split
 
 
def getCalssName(classNo):
    if   classNo == 0: return 'Speed Limit 20 km/h'
    elif classNo == 1: return 'Speed Limit 30 km/h'
    elif classNo == 2: return 'Speed Limit 50 km/h'
    elif classNo == 3: return 'Speed Limit 60 km/h'
    elif classNo == 4: return 'Speed Limit 70 km/h'
    elif classNo == 5: return 'Speed Limit 80 km/h'
    elif classNo == 6: return 'End of Speed Limit 80 km/h'
    elif classNo == 7: return 'Speed Limit 100 km/h'
    elif classNo == 8: return 'Speed Limit 120 km/h'
    elif classNo == 9: return 'No passing'
    elif classNo == 10: return 'No passing for vechiles over 3.5 metric tons'
    elif classNo == 11: return 'Right-of-way at the next intersection'
    elif classNo == 12: return 'Priority road'
    elif classNo == 13: return 'Yield'
    elif classNo == 14: return 'Stop'
    elif classNo == 15: return 'No vechiles'
    elif classNo == 16: return 'Vechiles over 3.5 metric tons prohibited'
    elif classNo == 17: return 'No entry'
    elif classNo == 18: return 'General caution'
    elif classNo == 19: return 'Dangerous curve to the left'
    elif classNo == 20: return 'Dangerous curve to the right'
    elif classNo == 21: return 'Double curve'
    elif classNo == 22: return 'Bumpy road'
    elif classNo == 23: return 'Slippery road'
    elif classNo == 24: return 'Road narrows on the right'
    elif classNo == 25: return 'Road work'
    elif classNo == 26: return 'Traffic signals'
    elif classNo == 27: return 'Pedestrians'
    elif classNo == 28: return 'Children crossing'
    elif classNo == 29: return 'Bicycles crossing'
    elif classNo == 30: return 'Beware of ice/snow'
    elif classNo == 31: return 'Wild animals crossing'
    elif classNo == 32: return 'End of all speed and passing limits'
    elif classNo == 33: return 'Turn right ahead'
    elif classNo == 34: return 'Turn left ahead'
    elif classNo == 35: return 'Ahead only'
    elif classNo == 36: return 'Go straight or right'
    elif classNo == 37: return 'Go straight or left'
    elif classNo == 38: return 'Keep right'
    elif classNo == 39: return 'Keep left'
    elif classNo == 40: return 'Roundabout mandatory'
    elif classNo == 41: return 'End of no passing'
    elif classNo == 42: return 'End of no passing by vechiles over 3.5 metric tons'
 
 
def grayscale(img):
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    return img
 
def equalize(img):
    img =cv2.equalizeHist(img)
    return img
 
def preprocessing(img):
    # img = grayscale(img)
    img = equalize(img)
    img = img/255
    return img
 
def read_imgs(image_dir, label=0):
    # 读取图片
    image_files = os.listdir(image_dir)
    images = []
    labels = []
    for image_file in image_files:
        image_path = os.path.join(image_dir, image_file)
        image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)  # 以灰度模式读取图片
        image = cv2.resize(image, (30, 30)) 
        img = preprocessing(image)
        images.append(img)
        labels.append(label)
    return images, labels
 
def test_predict(model):
    # 读取图片
    image_dir = 'D:/welink/STOP/before'
    # images = read_imgs(image_dir)
    image_files = os.listdir(image_dir)
    images = []
    for image_file in image_files:
        image_path = os.path.join(image_dir, image_file)
        image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)  # 以灰度模式读取图片
        image = cv2.resize(image, (30, 30)) 
        img = preprocessing(image)
        img = img.reshape(1, 30, 30, 1)
        images.append(img)
        # PREDICT IMAGE
        predictions = model.predict(img)
        predict_x=model.predict(img)
        classIndex=np.argmax(predict_x)
        probabilityValue =np.amax(predictions)
        print("img path:", image_file, " ==> ", str(classIndex)+" "+str(getCalssName(classIndex)))
        print(str(round(probabilityValue*100,2) )+"%")
 
def refact_model(model):
    base_model = model
    # 移除最后的分类层
    base_model = Model(inputs=base_model.input, outputs=base_model.layers[-2].output)
    # 添加一个新的分类层
    output = Dense(2, activation='softmax', name='new_dense')(base_model.output)
    model = Model(inputs=base_model.input, outputs=output)
    # 编译模型
    model.compile(optimizer=Adam(), loss='categorical_crossentropy', metrics=['accuracy'])
    return model
 
def retrain_with2label(model):
    image_dir1 = 'D:/welink/STOP/14'
    image_dir2 = 'D:/welink/STOP/17'
    images1, labels1 = read_imgs(image_dir1, 0)
    images2, labels2 = read_imgs(image_dir2, 1)
 
    # 合并图片和标签
    images = images1 + images2
    labels = labels1 + labels2
 
    images = np.array(images, dtype='float32')
    # 如果模型的输入形状是(30, 30, 1)，那么我们需要增加一个维度
    if model.input_shape[-1] == 1:
        images = np.expand_dims(images, axis=-1)
 
    labels = np.array(labels)
    labels = to_categorical(labels, num_classes=2)
 
    # 划分训练集和测试集
    train_images, test_images, train_labels, test_labels = train_test_split(images, labels, test_size=0.2)
    # 训练模型
    model.fit(train_images, train_labels, validation_data=(test_images, test_labels), epochs=10)
 
def test_predict2(model):
    # 选择stop的图像，扰动前的
    images, _ = read_imgs('D:/welink/STOP/before')
    if model.input_shape[-1] == 1:
        images = np.expand_dims(images, axis=-1)
    preds = model.predict(images)
    print('Predicted before:', preds.argmax(axis=1))
    return images
 
def run_art(images):
    # 创建一个目标标签（我们希望模型将0 stop识别为1 no entry）
    target_label = to_categorical(1, num_classes=2)
    target_label = np.tile(target_label, (len(images), 1))
 
    # 创建ART分类器
    classifier = TensorFlowV2Classifier(
        model=model,
        nb_classes=2,
        input_shape=(30, 30, 1),
        loss_object=tf.keras.losses.CategoricalCrossentropy(from_logits=True),
        clip_values=(0, 1)
    )
    # 创建FGSM实例
    attack = FastGradientMethod(estimator=classifier, targeted=True)
 
    # 初始化对抗样本为原始图像
    adv_images = np.copy(images)
 
    for i in range(100):  # 最多迭代100次
        # 生成对抗样本的扰动
        perturbations = attack.generate(x=adv_images, y=target_label) - adv_images
 
        # 计算所有样本的平均扰动
        avg_perturbation = np.mean(perturbations, axis=0)
 
        # 将平均扰动添加到所有对抗样本上
        adv_images += avg_perturbation
 
        # 使用模型对对抗样本进行预测
        preds = model.predict(adv_images)
        print('Iteration:', i, 'Predicted after:', preds.argmax(axis=1))
 
        # 如果所有的预测结果都为1，那么停止迭代
        if np.all(preds.argmax(axis=1) == 1):
            break
 
    # 保存对抗样本
    for i in range(len(adv_images)):
        # 将图像的数据类型转换为uint8，并且将图像的值范围调整到[0, 255]
        img = (adv_images[i] * 255).astype(np.uint8)
        # 保存图像
        cv2.imwrite(f'traffic_adv_image_{i}.png', img)
 
    # 归一化平均扰动并保存为图像
    avg_perturbation = (avg_perturbation - np.min(avg_perturbation)) / (np.max(avg_perturbation) - np.min(avg_perturbation))
    # 将平均扰动的值范围调整到[0, 255]，并转换为uint8类型
    avg_perturbation = (avg_perturbation * 255).astype(np.uint8)
    # 将灰度图像转换为RGB图像
    avg_perturbation_rgb = cv2.cvtColor(avg_perturbation, cv2.COLOR_GRAY2RGB)
    # 保存图像
    cv2.imwrite('traffic_avg_perturbation.png', avg_perturbation_rgb)
 
 
if __name__ == "__main__":
    # 找到一个训练好的，识别交通信号牌的模型: https://github.com/Daulettulegenov/TSR_CNN
    model = load_model(r'D:\source\competition\TSR_CNN-main\CNN_model_3.h5')
    # 预测原始的输出类型，可以看到并不能正确的分类，因为是中文字幕 停！！！！而不是 STOP
    test_predict(model)
    # 因此，需要迁移训练，让其识别中文的“停”
    model = refact_model(model)
    # 测试是否可以识别中文的停
    retrain_with2label(model)
    # 预测新的输出类型，可以看到能正确的分类，即便是中文的停！！！
    images = test_predict2(model)
    # 生成扰动图像，让其扰动，识别为no entry，保存扰动图像
    run_art(images)

效果：

Predicted before: [0 0 0 0 0 0 0 0 0 0]

output, from_logits = _get_logits(....

1/1 [==============================] - 0s 38ms/step

Iteration: 0 Predicted after: [1 1 0 1 0 0 1 0 1 0]

1/1 [==============================] - 0s 46ms/step

Iteration: 1 Predicted after: [1 1 1 1 1 1 1 1 1 1]

生成的对抗样本：

依稀可见“停”

扰动图像：

代码说明：这份代码的主要目的是使用深度学习模型来识别交通信号牌，并使用对抗性攻击来生成能够误导模型的对抗样本。

以下是代码的主要部分的解释：

1. getCalssName(classNo): 这个函数根据类别编号返回对应的交通信号牌的名称。

2. grayscale(img), equalize(img), preprocessing(img): 这些函数用于图像预处理，包括将图像转换为灰度图像，进行直方图均衡化，以及归一化。

3. read_imgs(image_dir, label=0): 这个函数读取指定目录下的所有图像，并返回图像数据和对应的标签。

4. test_predict(model): 这个函数读取指定目录下的所有图像，然后使用模型进行预测，并打印出预测结果。

5. refact_model(model): 这个函数修改模型的结构，移除最后的分类层，然后添加一个新的分类层。

6. retrain_with2label(model): 这个函数读取两个目录下的所有图像，然后使用这些图像和对应的标签来训练模型。

7. test_predict2(model): 这个函数读取指定目录下的所有图像，然后使用模型进行预测，并返回预测结果和图像数据。

8. run_art(images): 这个函数使用对抗性攻击来生成对抗样本，并保存对抗样本和平均扰动。

在if __name__ == "__main__"部分，代码首先加载一个预训练的模型，然后使用这个模型来预测原始的交通信号牌图像。然后，代码修改模型的结构，并使用新的数据来训练模型。最后，代码使用新的模型来预测原始的交通信号牌图像，并使用对抗性攻击来生成对抗样本。

样本下载：

交通信号灯 https://www.kaggle.com/datasets/meowmeowmeowmeowmeow/gtsrb-german-traffic-sign