From 54953ec21fc8373ea798f85e507a4c88d0b6ccbe Mon Sep 17 00:00:00 2001 From: MahouShoujoMivutilde Date: Thu, 13 Jun 2024 01:41:21 +0300 Subject: [PATCH 1/5] Use 32KB buffer for `copyFile()` (reduces copy time by 30%) This makes `:paste` comparable in performance to `cp --reflink=never` for large files. For large number of small files improvements are less substantial (compared to `cp -r --reflink=never`), though still noticeable. In both cases the copy takes about 30% less time than with `buf` size at 4096. 32KB is the same number `io.Copy()` uses internally (when it can), and is about where improvements stop. Context: https://github.com/gokcehan/lf/issues/1685#issuecomment-2163934296 --- copy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copy.go b/copy.go index cec83c28..59a9a406 100644 --- a/copy.go +++ b/copy.go @@ -47,7 +47,7 @@ func copyFile(src, dst string, preserve []string, info os.FileInfo, nums chan in } } - buf := make([]byte, 4096) + buf := make([]byte, 32*1024) r, err := os.Open(src) if err != nil { From ed7ee991f372b8b8b4297417017c13a9f5166a95 Mon Sep 17 00:00:00 2001 From: MahouShoujoMivutilde Date: Fri, 14 Jun 2024 03:53:40 +0300 Subject: [PATCH 2/5] Apply suggested changes --- copy.go | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/copy.go b/copy.go index 59a9a406..06f2cb88 100644 --- a/copy.go +++ b/copy.go @@ -11,6 +11,24 @@ import ( "github.com/djherbis/times" ) +type ProgressWriter struct { + writer io.Writer + nums chan<- int64 +} + +func NewProgressWriter(writer io.Writer, nums chan<- int64) *ProgressWriter { + return &ProgressWriter{ + writer: writer, + nums: nums, + } +} + +func (progressWriter *ProgressWriter) Write(b []byte) (int, error) { + n, err := progressWriter.writer.Write(b) + progressWriter.nums <- int64(n) + return n, err +} + func copySize(srcs []string) (int64, error) { var total int64 @@ -47,8 +65,6 @@ func copyFile(src, dst string, preserve []string, info os.FileInfo, nums chan in } } - buf := make([]byte, 32*1024) - r, err := os.Open(src) if err != nil { return err @@ -60,23 +76,13 @@ func copyFile(src, dst string, preserve []string, info os.FileInfo, nums chan in return err } - for { - n, err := r.Read(buf) - if err != nil && err != io.EOF { - w.Close() - os.Remove(dst) - return err - } - - if n == 0 { - break - } - - if _, err := w.Write(buf[:n]); err != nil { - return err - } - - nums <- int64(n) + _, err = io.Copy(NewProgressWriter(w, nums), r) + if err == io.ErrShortWrite { + return err + } else if err != nil { + w.Close() + os.Remove(dst) + return err } if err := w.Close(); err != nil { From 7f681cace52b4a6ef6e7a06fc7b47d4daa59812e Mon Sep 17 00:00:00 2001 From: MahouShoujoMivutilde Date: Fri, 14 Jun 2024 05:45:05 +0300 Subject: [PATCH 3/5] Just delete incomplete an file on all errors --- copy.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/copy.go b/copy.go index 06f2cb88..336e5f9a 100644 --- a/copy.go +++ b/copy.go @@ -77,9 +77,7 @@ func copyFile(src, dst string, preserve []string, info os.FileInfo, nums chan in } _, err = io.Copy(NewProgressWriter(w, nums), r) - if err == io.ErrShortWrite { - return err - } else if err != nil { + if err != nil { w.Close() os.Remove(dst) return err From 7487704ddf49db0ed1eba3475736c1769e50e4d9 Mon Sep 17 00:00:00 2001 From: MahouShoujoMivutilde Date: Fri, 14 Jun 2024 07:09:16 +0300 Subject: [PATCH 4/5] Mention new progress UI update frequency --- app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.go b/app.go index a95b1801..6314db05 100644 --- a/app.go +++ b/app.go @@ -332,7 +332,7 @@ func (app *app) loop() { return case n := <-app.nav.copyBytesChan: app.nav.copyBytes += n - // n is usually 4096B so update roughly per 4096B x 1024 = 4MB copied + // n is usually 32*1024B (default io.Copy() buffer) so update roughly per 32KB x 1024 = 32MB copied if app.nav.copyUpdate++; app.nav.copyUpdate >= 1024 { app.nav.copyUpdate = 0 app.ui.draw(app.nav) From f5d43fe3ddfdba349c9611f1b0a1fc7921df3889 Mon Sep 17 00:00:00 2001 From: MahouShoujoMivutilde Date: Fri, 14 Jun 2024 07:49:20 +0300 Subject: [PATCH 5/5] Return the old (each 4MB copied) progress update frequency (progress UI updates don't seem to impact performance much) --- app.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.go b/app.go index 6314db05..87b70574 100644 --- a/app.go +++ b/app.go @@ -332,8 +332,8 @@ func (app *app) loop() { return case n := <-app.nav.copyBytesChan: app.nav.copyBytes += n - // n is usually 32*1024B (default io.Copy() buffer) so update roughly per 32KB x 1024 = 32MB copied - if app.nav.copyUpdate++; app.nav.copyUpdate >= 1024 { + // n is usually 32*1024B (default io.Copy() buffer) so update roughly per 32KB x 128 = 4MB copied + if app.nav.copyUpdate++; app.nav.copyUpdate >= 128 { app.nav.copyUpdate = 0 app.ui.draw(app.nav) }