Where the classifier looks

import numpy as np

def softmax(z):
    e = np.exp(z - z.max())
    return e / e.sum()

# A toy classifier for 4 x 4 pictures with two labels. "Stripe" scores the
# brightest row; "blob" scores the middle four pixels. Pixels are already
# normalized, so 0 is the average color, the gray a covered square gets.
def probs(img):
    stripe = 3 * img.sum(axis=1).max() / 4
    blob = 3 * img[1:3, 1:3].mean()
    return softmax(np.array([stripe, blob]))

def occlusion_map(img, label, size=2):
    base = probs(img)[label]
    n = img.shape[0] // size
    drops = np.zeros((n, n))
    for r in range(n):
        for c in range(n):
            covered = img.copy()
            covered[r*size:(r+1)*size, c*size:(c+1)*size] = 0  # gray square
            drops[r, c] = base - probs(covered)[label]
    return base, drops

img = np.zeros((4, 4))
img[0, :] = 1.0            # a bright stripe along the top
base, drops = occlusion_map(img, label=0)
print(f"stripe with nothing covered: {base:.2f}")
print("drop when each 2 x 2 square is covered:")
print(np.round(drops, 2))

# The same evidence twice: a second stripe along the bottom. The classifier
# takes the brightest row, so covering either stripe leaves the other.
img[3, :] = 1.0
base, drops = occlusion_map(img, label=0)
print(f"two stripes, nothing covered: {base:.2f}")
print(np.round(drops, 2))

stripe with nothing covered: 0.95
drop when each 2 x 2 square is covered:
[[0.13 0.13]
 [0.   0.  ]]
two stripes, nothing covered: 0.95
[[0. 0.]
 [0. 0.]]