diff --git a/src/IRVisitor.cpp b/src/IRVisitor.cpp
index 3db88dab257e..9a5e6a8e0537 100644
--- a/src/IRVisitor.cpp
+++ b/src/IRVisitor.cpp
@@ -270,18 +270,27 @@ void IRVisitor::visit(const HoistedStorage *op) {
 }
 
 void IRGraphVisitor::include(const Expr &e) {
-    auto r = visited.insert(e.get());
-    if (r.second) {
-        // Was newly inserted
+    if (e.is_sole_reference()) {
+        // We can't have visited it before, and won't visit it again.
         e.accept(this);
+    } else {
+        auto r = visited.insert(e.get());
+        if (r.second) {
+            // Was newly inserted
+            e.accept(this);
+        }
     }
 }
 
 void IRGraphVisitor::include(const Stmt &s) {
-    auto r = visited.insert(s.get());
-    if (r.second) {
-        // Was newly inserted
+    if (s.is_sole_reference()) {
         s.accept(this);
+    } else {
+        auto r = visited.insert(s.get());
+        if (r.second) {
+            // Was newly inserted
+            s.accept(this);
+        }
     }
 }
 
diff --git a/src/IRVisitor.h b/src/IRVisitor.h
index 4de6cd2dc8a9..4be3b82a1454 100644
--- a/src/IRVisitor.h
+++ b/src/IRVisitor.h
@@ -94,8 +94,10 @@ class IRGraphVisitor : public IRVisitor {
     // @}
 
 private:
-    /** The nodes visited so far */
-    std::set<IRHandle> visited;
+    /** The nodes visited so far. Only includes nodes with a ref count greater
+     * than one, because we know that nodes with a ref count of 1 will only be
+     * visited once if their parents are only visited once. */
+    std::set<const IRNode *> visited;
 
 protected:
     /** These methods should call 'include' on the children to only