diff --git a/mem/buffers.go b/mem/buffers.go
index 1e3d696cbcd5..b1f7653a4c97 100644
--- a/mem/buffers.go
+++ b/mem/buffers.go
@@ -110,7 +110,7 @@ func NewBuffer(data *[]byte, pool BufferPool) Buffer {
 	b.data = *data
 	b.pool = pool
 	b.refs = refObjectPool.Get().(*atomic.Int32)
-	b.refs.Add(1)
+	b.Ref()
 	return b
 }
 
@@ -140,9 +140,7 @@ func (b *buffer) ReadOnlyData() []byte {
 }
 
 func (b *buffer) Ref() {
-	if b.tracking {
-		b.refCallers = append(b.refCallers, string(debug.Stack()))
-	}
+	b.refCallers = append(b.refCallers, string(debug.Stack()))
 	if b.refs == nil {
 		panic("Cannot ref freed buffer")
 	}
@@ -154,9 +152,7 @@ func (b *buffer) Track() {
 }
 
 func (b *buffer) Free() {
-	if b.tracking {
-		b.freeCallers = append(b.freeCallers, string(debug.Stack()))
-	}
+	b.freeCallers = append(b.freeCallers, string(debug.Stack()))
 	if b.refs == nil {
 		panic("Cannot free freed buffer")
 	}
@@ -201,7 +197,7 @@ func (b *buffer) split(n int) (Buffer, Buffer) {
 		panic("Cannot split freed buffer")
 	}
 
-	b.refs.Add(1)
+	b.Ref()
 	split := newBuffer()
 	split.origData = b.origData
 	split.data = b.data[n:]