-
Notifications
You must be signed in to change notification settings - Fork 117
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ark will test uploading a single object that is copied to multiple individual objects. This feature is only available on a recent MinIO server. Parameters: * `--obj.size=N` controls the size of each object that is uploaded. Default is 1MiB. * `--copies=N` controls the number of objects per TAR file. Default is 100. Size is calculated as `--obj.size` * `--copies`. Example: Use 8 concurrent uploads to copy a 512KB objects to 50 locations. ``` λ warp fanout --copies=50 --obj.size=512KiB --concurrent=8 --duration=1m warp: Benchmark data written to "warp-fanout-2023-06-15[105151]-j3qb.csv.zst" ---------------------------------------- Operation: POST * Average: 113.06 MiB/s, 226.12 obj/s Throughput, split into 57 x 1s: * Fastest: 178.4MiB/s, 356.74 obj/s * 50% Median: 113.9MiB/s, 227.76 obj/s * Slowest: 56.3MiB/s, 112.53 obj/s warp: Cleanup Done. ``` The analysis throughput represents the object count and sizes as they are written when extracted. Request times shown with `--analyze.v` represents request time for each fan-out call.
- Loading branch information
Showing
6 changed files
with
259 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -97,6 +97,7 @@ func init() { | |
| multipartCmd, | ||
| zipCmd, | ||
| snowballCmd, | ||
| fanoutCmd, | ||
| } | ||
| b := []cli.Command{ | ||
| analyzeCmd, | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| /* | ||
| * Warp (C) 2019-2023 MinIO, Inc. | ||
| * | ||
| * This program is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU Affero General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU Affero General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU Affero General Public License | ||
| * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
|
|
||
| package cli | ||
|
|
||
| import ( | ||
| "github.com/minio/cli" | ||
| "github.com/minio/pkg/console" | ||
| "github.com/minio/warp/pkg/bench" | ||
| ) | ||
|
|
||
| var fanoutFlags = []cli.Flag{ | ||
| cli.StringFlag{ | ||
| Name: "obj.size", | ||
| Value: "1MiB", | ||
| Usage: "Size of each generated object. Can be a number or 10KiB/MiB/GiB. All sizes are base 2 binary.", | ||
| }, | ||
| cli.IntFlag{ | ||
| Name: "copies", | ||
| Value: 100, | ||
| Usage: "Number of copies per uploaded object", | ||
| Hidden: true, | ||
| }, | ||
| } | ||
|
|
||
| // Fanout command. | ||
| var fanoutCmd = cli.Command{ | ||
| Name: "fanout", | ||
| Usage: "benchmark fan-out of objects on MinIO servers", | ||
| Action: mainFanout, | ||
| Before: setGlobalsFromContext, | ||
| Flags: combineFlags(globalFlags, ioFlags, fanoutFlags, genFlags, benchFlags, analyzeFlags), | ||
| CustomHelpTemplate: `NAME: | ||
| {{.HelpName}} - {{.Usage}} | ||
| USAGE: | ||
| {{.HelpName}} [FLAGS] | ||
| -> see https://github.com/minio/warp#fanout | ||
| FLAGS: | ||
| {{range .VisibleFlags}}{{.}} | ||
| {{end}}`, | ||
| } | ||
|
|
||
| // mainFanout is the entry point for cp command. | ||
| func mainFanout(ctx *cli.Context) error { | ||
| checkFanoutSyntax(ctx) | ||
| b := bench.Fanout{ | ||
| Copies: ctx.Int("copies"), | ||
| Common: getCommon(ctx, newGenSource(ctx, "obj.size")), | ||
| } | ||
| return runBench(ctx, &b) | ||
| } | ||
|
|
||
| func checkFanoutSyntax(ctx *cli.Context) { | ||
| if ctx.NArg() > 0 { | ||
| console.Fatal("Command takes no arguments") | ||
| } | ||
| if ctx.Int("copies") <= 0 { | ||
| console.Fatal("Copies must be bigger than 0") | ||
| } | ||
| checkAnalyze(ctx) | ||
| checkBenchmark(ctx) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| /* | ||
| * Warp (C) 2019-2023 MinIO, Inc. | ||
| * | ||
| * This program is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU Affero General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU Affero General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU Affero General Public License | ||
| * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| */ | ||
|
|
||
| package bench | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "net/http" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "github.com/minio/minio-go/v7" | ||
| ) | ||
|
|
||
| // Fanout benchmarks upload speed. | ||
| type Fanout struct { | ||
| Common | ||
| Copies int | ||
| prefixes map[string]struct{} | ||
| } | ||
|
|
||
| // Prepare will create an empty bucket ot delete any content already there. | ||
| func (u *Fanout) Prepare(ctx context.Context) error { | ||
| return u.createEmptyBucket(ctx) | ||
| } | ||
|
|
||
| // Start will execute the main benchmark. | ||
| // Operations should begin executing when the start channel is closed. | ||
| func (u *Fanout) Start(ctx context.Context, wait chan struct{}) (Operations, error) { | ||
| var wg sync.WaitGroup | ||
| wg.Add(u.Concurrency) | ||
| u.addCollector() | ||
| c := u.Collector | ||
| if u.AutoTermDur > 0 { | ||
| ctx = c.AutoTerm(ctx, http.MethodPost, u.AutoTermScale, autoTermCheck, autoTermSamples, u.AutoTermDur) | ||
| } | ||
| u.prefixes = make(map[string]struct{}, u.Concurrency) | ||
|
|
||
| // Non-terminating context. | ||
| nonTerm := context.Background() | ||
|
|
||
| for i := 0; i < u.Concurrency; i++ { | ||
| src := u.Source() | ||
| u.prefixes[src.Prefix()] = struct{}{} | ||
| go func(i int) { | ||
| rcv := c.Receiver() | ||
| defer wg.Done() | ||
| opts := minio.PutObjectFanOutRequest{ | ||
| Entries: make([]minio.PutObjectFanOutEntry, u.Copies), | ||
| Checksum: minio.Checksum{}, | ||
| SSE: nil, | ||
| } | ||
| done := ctx.Done() | ||
|
|
||
| <-wait | ||
| for { | ||
| select { | ||
| case <-done: | ||
| return | ||
| default: | ||
| } | ||
| obj := src.Object() | ||
| for i := range opts.Entries { | ||
| opts.Entries[i] = minio.PutObjectFanOutEntry{ | ||
| Key: fmt.Sprintf("%s/copy-%d.ext", obj.Name, i), | ||
| UserMetadata: u.PutOpts.UserMetadata, | ||
| UserTags: u.PutOpts.UserTags, | ||
| ContentType: obj.ContentType, | ||
| ContentEncoding: u.PutOpts.ContentEncoding, | ||
| ContentDisposition: u.PutOpts.ContentDisposition, | ||
| ContentLanguage: u.PutOpts.ContentLanguage, | ||
| CacheControl: u.PutOpts.CacheControl, | ||
| } | ||
| } | ||
| client, cldone := u.Client() | ||
| op := Operation{ | ||
| OpType: http.MethodPost, | ||
| Thread: uint16(i), | ||
| Size: obj.Size * int64(u.Copies), | ||
| ObjPerOp: u.Copies, | ||
| File: obj.Name, | ||
| Endpoint: client.EndpointURL().String(), | ||
| } | ||
|
|
||
| op.Start = time.Now() | ||
| res, err := client.PutObjectFanOut(nonTerm, u.Bucket, obj.Reader, opts) | ||
| op.End = time.Now() | ||
| if err != nil { | ||
| u.Error("upload error: ", err) | ||
| op.Err = err.Error() | ||
| } | ||
|
|
||
| if len(res) != u.Copies && op.Err == "" { | ||
| err := fmt.Sprint("short upload. want:", u.Copies, " copies, got:", len(res)) | ||
| if op.Err == "" { | ||
| op.Err = err | ||
| } | ||
| u.Error(err) | ||
| } | ||
| cldone() | ||
| rcv <- op | ||
| } | ||
| }(i) | ||
| } | ||
| wg.Wait() | ||
| return c.Close(), nil | ||
| } | ||
|
|
||
| // Cleanup deletes everything uploaded to the bucket. | ||
| func (u *Fanout) Cleanup(ctx context.Context) { | ||
| pf := make([]string, 0, len(u.prefixes)) | ||
| for p := range u.prefixes { | ||
| pf = append(pf, p) | ||
| } | ||
| u.deleteAllInBucket(ctx, pf...) | ||
| } |