Grad-CAM_demo¶
Part of MSc project. Involves using Grad-CAM to visualize which regions of an image were important for classification
CLICK HERE to go to the repository
A jupyter notebook displaying a use of Grad-CAM to highlight areas of an image that were important for a prediction.
This technique was used with VGG16 and MobileNet to help interpret the performance of the CNNs.It helps to show how the models were making predictions. CNNs may learn unexpected features from the images in the dataset e.g. areas of the background rather than the subject of the image.
The example uploaded here uses VGG16.
Grad-CAM paper: Selvaraju, Ramprasaath R., Michael Cogswell, Abhishek Das, Ramakrishna Vedantam, Devi Parikh, and Dhruv Batra. 2019. “Grad-CAM: Visual Explanations from Deep Networks via Gradient-Based Localization.” International Journal of Computer Vision, October. https://doi.org/10.1007/s11263-019-01228-7.
The solution in this notebook is based on the implementations by:
- F. Chollet(2017) in his Deep Learning with Python book
- Rapahel Meudec's, writer of the tf-explain library, solution which is updated for TensorFlow 2.0: source
# Using Grad-CAM output for a classifier trained to identify Scorpion pepper images
import os
import tensorflow.keras.backend as K
import tensorflow as tf
from tensorflow.keras.models import Model
from tensorflow.keras.models import load_model
from tensorflow.keras.preprocessing import image
from PIL import Image
import cv2
import matplotlib.pyplot as plt
import numpy as np
def grad_cam(input_model, pred_class, layer_name, images):
#Reference: https://stackoverflow.com/questions/58322147/how-to-generate-cnn-heatmaps-using-built-in-keras-in-tf2-0-tf-keras
conv_output = input_model.get_layer(layer_name).output
# model with 2 outputs
heatmap_model = Model([input_model.inputs], [conv_output, input_model.output])
# Get gradient of the predicted result w.r.t. the output of the (last) conv. layer
with tf.GradientTape() as gtape:
conv_output, predictions = heatmap_model(images)
print(predictions)
result = predictions[0]
# 1 == Scorpion pepper, 0 == other
if pred_class == 0:
result = 1 - result
grads = gtape.gradient(result, conv_output)
pooled_grads = K.mean(grads, axis=(0, 1, 2))
#print(pooled_grads.shape)
heatmap = tf.reduce_mean(tf.multiply(pooled_grads, conv_output), axis=-1)
heatmap = np.maximum(heatmap, 0)
max_heat = np.max(heatmap)
if max_heat == 0:
max_heat = 1e-10
heatmap /= max_heat
heatmap = heatmap[0]
#print(heatmap.shape)
heatmap = Image.fromarray(heatmap)
heatmap = np.array(heatmap)
heatmap = cv2.resize(heatmap, (224,224), cv2.INTER_LINEAR)
#print(heatmap.shape)
return heatmap
def isScorpion(pred):
if pred<0.5:
return "not scorpion pepper"
return "scorpion pepper"
K.clear_session()
model = load_model("vgg16_test6_p1.h5")
labels = ["not_scorpion", "scorpion_pepper"]
#model.summary()
path = "gradcam_dataset/"
dataset = os.listdir(path)
# iterate through images in directory, generate Grad-CAM output, plot grid of the output images
for folder in dataset:
i = 1
columns = 3
rows = 5 #change back to 5
fig = plt.figure(figsize = (20, 20))
for file in os.listdir(path+folder):
img = image.load_img(path+folder+'\\'+file,target_size=(224, 224))
x = image.img_to_array(img)
# change x to appropriate shape: (1, 224, 224, 3)
x = np.expand_dims(x, axis=0)
# preprocessing step - rescale pixel color values between 0 and 1
x /=255
# change input shape and make prediction using model
images = np.vstack([x])
pred_y = model.predict(x, batch_size=1)
print(pred_y)
# Grad-CAM output for each class using layer: block5_conv3
cam = grad_cam(model, 0, "block5_conv3", images)
cam2 = grad_cam(model, 1, "block5_conv3", images)
ax0 = fig.add_subplot(rows,columns,i)
pred = float(pred_y[0][0])
pred = "{0:.5f}".format(pred)
ax0.set_xlabel('Class = {}, Score = {}'.format(isScorpion(pred_y[0][0]), pred))
ax0.imshow(img)
i = i+1
ax1 = fig.add_subplot(rows,columns,i)
pred = float(pred_y[0][0])
pred = "{0:.5f}".format(pred)
ax1.set_xlabel('GradCAM for not scorpion')
ax1.imshow(img)
ax1.imshow(cam, cmap='jet', alpha=0.5)
i = i+1
ax2 = fig.add_subplot(rows,columns, i)
pred = float(pred_y[0][0])
pred = "{0:.5f}".format(pred)
ax2.set_xlabel('GradCAM for scorpion')
ax2.imshow(img)
ax2.imshow(cam2, cmap='jet', alpha=0.5)
i=i+1
#note left cannot be >= right
fig.subplots_adjust(left=0, right=0.5)
del model
K.clear_session() #seems to help with gpu memory leak in tensorflow 2.0
plt.show()
