diff --git a/api/core/v1/pod_webhook.go b/api/core/v1/pod_webhook.go
index 71bb68b3..c9fea0c0 100644
--- a/api/core/v1/pod_webhook.go
+++ b/api/core/v1/pod_webhook.go
@@ -126,8 +126,6 @@ func (a *ImageRewriter) handleContainer(pod *corev1.Pod, container *corev1.Conta
 		} // ignore rewriting invalid images
 	}
 
-	pod.Annotations[annotationKey] = image
-
 	if !rewriteImage {
 		return RewrittenImage{
 			Original:            container.Image,
@@ -135,6 +133,8 @@ func (a *ImageRewriter) handleContainer(pod *corev1.Pod, container *corev1.Conta
 		}
 	}
 
+	pod.Annotations[annotationKey] = image
+
 	sanitizedRegistryName := strings.ReplaceAll(sourceRef.Context().RegistryStr(), ":", "-")
 	image = strings.ReplaceAll(image, sourceRef.Context().RegistryStr(), sanitizedRegistryName)