Merge branch 'master' into i256-pgsidecar--wip

cmd/datamon/cmd/bundle.go

@@ -5,6 +5,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"text/template"
 
 	"github.com/oneconcern/datamon/pkg/core"
 	"github.com/oneconcern/datamon/pkg/storage"
@@ -22,10 +23,17 @@ Every bundle is an entry in the history of a repository at a point in time.
 `,
 }
 
+var bundleDescriptorTemplate *template.Template
+
 func init() {
 	rootCmd.AddCommand(bundleCmd)
 	addBucketNameFlag(bundleCmd)
 	addBlobBucket(bundleCmd)
+
+	bundleDescriptorTemplate = func() *template.Template {
+		const listLineTemplateString = `{{.ID}} , {{.Timestamp}} , {{.Message}}`
+		return template.Must(template.New("list line").Parse(listLineTemplateString))
+	}()
 }
 
 func setLatestOrLabelledBundle(ctx context.Context, store storage.Store) error {

cmd/datamon/cmd/bundle_get.go

@@ -0,0 +1,80 @@
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/oneconcern/datamon/pkg/core"
+
+	"github.com/spf13/cobra"
+	"golang.org/x/sys/unix"
+)
+
+var GetBundleCommand = &cobra.Command{
+	Use:   "get",
+	Short: "Get bundle info by id",
+	Long: `Performs a direct lookup of labels by id.
+Prints corresponding bundle information if the label exists,
+exits with ENOENT status otherwise.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		ctx := context.Background()
+		remoteStores, err := paramsToRemoteCmdStores(ctx, params)
+		if err != nil {
+			logFatalln(err)
+			return
+		}
+
+		err = setLatestOrLabelledBundle(ctx, remoteStores.meta)
+		if err == core.ErrNotFound {
+			fmt.Fprintf(os.Stderr, "didn't find label %q\n", params.label.Name)
+			osExit(int(unix.ENOENT))
+			return
+		}
+		if err != nil {
+			logFatalln(err)
+			return
+		}
+		bundle := core.New(core.NewBDescriptor(),
+			core.Repo(params.repo.RepoName),
+			core.MetaStore(remoteStores.meta),
+			core.BundleID(params.bundle.ID),
+		)
+
+		err = core.DownloadMetadata(ctx, bundle)
+		if err == core.ErrNotFound {
+			fmt.Fprintf(os.Stderr, "didn't find bundle '%v'\n", params.bundle.ID)
+			osExit(int(unix.ENOENT))
+			return
+		}
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			logFatalf("error downloading bundle information\n")
+		}
+
+		var buf bytes.Buffer
+		err = bundleDescriptorTemplate.Execute(&buf, bundle.BundleDescriptor)
+		if err != nil {
+			log.Println("executing template:", err)
+		}
+		log.Println(buf.String())
+	},
+}
+
+func init() {
+	requiredFlags := []string{addRepoNameOptionFlag(GetBundleCommand)}
+
+	addBundleFlag(GetBundleCommand)
+	addLabelNameFlag(GetBundleCommand)
+
+	for _, flag := range requiredFlags {
+		err := GetBundleCommand.MarkFlagRequired(flag)
+		if err != nil {
+			logFatalln(err)
+		}
+	}
+
+	bundleCmd.AddCommand(GetBundleCommand)
+}

cmd/datamon/cmd/bundle_list.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"log"
-	"text/template"
 
 	"github.com/oneconcern/datamon/pkg/core"
 
@@ -16,9 +15,7 @@ var BundleListCommand = &cobra.Command{
 	Short: "List bundles",
 	Long:  "List the bundles in a repo",
 	Run: func(cmd *cobra.Command, args []string) {
-		const listLineTemplateString = `{{.ID}} , {{.Timestamp}} , {{.Message}}`
 		ctx := context.Background()
-		listLineTemplate := template.Must(template.New("list line").Parse(listLineTemplateString))
 		remoteStores, err := paramsToRemoteCmdStores(ctx, params)
 		if err != nil {
 			logFatalln(err)
@@ -29,7 +26,7 @@ var BundleListCommand = &cobra.Command{
 		}
 		for _, bd := range bundleDescriptors {
 			var buf bytes.Buffer
-			err := listLineTemplate.Execute(&buf, bd)
+			err := bundleDescriptorTemplate.Execute(&buf, bd)
 			if err != nil {
 				log.Println("executing template:", err)
 			}

cmd/datamon/cmd/cli_test.go

@@ -342,6 +342,29 @@ func TestRepoList(t *testing.T) {
 	require.True(t, testNow.Sub(ll[1].time).Seconds() < 3, "second timestamp bounds after second create")
 }
 
+func TestGetRepo(t *testing.T) {
+	cleanup := setupTests(t)
+	defer cleanup()
+	repoName := internal.RandStringBytesMaskImprSrc(8)
+	runCmd(t, []string{"repo",
+		"get",
+		"--repo", repoName,
+	}, "attempt get non-exist repo", true)
+	require.Equal(t, int(unix.ENOENT), exitMocks.exitStatuses[len(exitMocks.exitStatuses)-1],
+		"ENOENT on nonexistant label")
+	runCmd(t, []string{"repo",
+		"create",
+		"--description", "testing",
+		"--repo", repoName,
+		"--name", "tests",
+		"--email", "[email protected]",
+	}, "create second test repo", false)
+	runCmd(t, []string{"repo",
+		"get",
+		"--repo", repoName,
+	}, "get repo", false)
+}
+
 func testUploadBundle(t *testing.T, file uploadTree) {
 	r, w, err := os.Pipe()
 	if err != nil {
@@ -564,6 +587,41 @@ func TestListBundles(t *testing.T) {
 	}
 }
 
+func TestGetBundle(t *testing.T) {
+	cleanup := setupTests(t)
+	defer cleanup()
+	runCmd(t, []string{"repo",
+		"create",
+		"--description", "testing",
+		"--repo", repo1,
+		"--name", "tests",
+		"--email", "[email protected]",
+	}, "create second test repo", false)
+	label := internal.RandStringBytesMaskImprSrc(8)
+	runCmd(t, []string{"bundle",
+		"get",
+		"--repo", repo1,
+		"--label", label,
+	}, "attempt get non-exist bundle", true)
+	require.Equal(t, int(unix.ENOENT), exitMocks.exitStatuses[len(exitMocks.exitStatuses)-1],
+		"ENOENT on nonexistant bundle")
+	files := testUploadTrees[0]
+	file := files[0]
+	runCmd(t, []string{"bundle",
+		"upload",
+		"--path", dirPathStr(t, file),
+		"--message", "label test bundle",
+		"--repo", repo1,
+		"--label", label,
+		"--concurrency-factor", concurrencyFactor,
+	}, "upload bundle at "+dirPathStr(t, file), false)
+	runCmd(t, []string{"bundle",
+		"get",
+		"--repo", repo1,
+		"--label", label,
+	}, "get bundle", false)
+}
+
 func TestGetLabel(t *testing.T) {
 	cleanup := setupTests(t)
 	defer cleanup()
@@ -579,7 +637,7 @@ func TestGetLabel(t *testing.T) {
 		"get",
 		"--repo", repo1,
 		"--label", label,
-	}, "list labels", true)
+	}, "attempt get non-exist label", true)
 	require.Equal(t, int(unix.ENOENT), exitMocks.exitStatuses[len(exitMocks.exitStatuses)-1],
 		"ENOENT on nonexistant label")
 	files := testUploadTrees[0]
@@ -596,7 +654,7 @@ func TestGetLabel(t *testing.T) {
 		"get",
 		"--repo", repo1,
 		"--label", label,
-	}, "list labels", false)
+	}, "get label", false)
 }
 
 type labelListEntry struct {

cmd/datamon/cmd/repo.go

@@ -3,6 +3,8 @@
 package cmd
 
 import (
+	"text/template"
+
 	"github.com/spf13/cobra"
 )
 
@@ -16,6 +18,13 @@ Repos are datasets that are versioned and managed via bundles.
 `,
 }
 
+var repoDescriptorTemplate *template.Template
+
 func init() {
 	rootCmd.AddCommand(repoCmd)
+
+	repoDescriptorTemplate = func() *template.Template {
+		const listLineTemplateString = `{{.Name}} , {{.Description}} , {{with .Contributor}}{{.Name}} , {{.Email}}{{end}} , {{.Timestamp}}`
+		return template.Must(template.New("list line").Parse(listLineTemplateString))
+	}()
 }

cmd/datamon/cmd/repo_get.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/oneconcern/datamon/pkg/core"
+
+	"github.com/spf13/cobra"
+	"golang.org/x/sys/unix"
+)
+
+var GetRepoCommand = &cobra.Command{
+	Use:   "get",
+	Short: "Get repo info by name",
+	Long: `Performs a direct lookup of repos by name.
+Prints corresponding repo information if the name exists,
+exits with ENOENT status otherwise.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		ctx := context.Background()
+		remoteStores, err := paramsToRemoteCmdStores(ctx, params)
+		if err != nil {
+			logFatalln(err)
+		}
+		repoDescriptor, err := core.GetRepoDescriptorByRepoName(
+			remoteStores.meta, params.repo.RepoName)
+		if err == core.ErrNotFound {
+			fmt.Fprintf(os.Stderr, "didn't find repo '%v'\n", params.repo.RepoName)
+			osExit(int(unix.ENOENT))
+			return
+		}
+		if err != nil {
+			logFatalf("error downloading repo information: %v\n", err)
+		}
+
+		var buf bytes.Buffer
+		err = repoDescriptorTemplate.Execute(&buf, repoDescriptor)
+		if err != nil {
+			log.Println("executing template:", err)
+		}
+		log.Println(buf.String())
+	},
+}
+
+func init() {
+	requiredFlags := []string{addRepoNameOptionFlag(GetRepoCommand)}
+
+	for _, flag := range requiredFlags {
+		err := GetRepoCommand.MarkFlagRequired(flag)
+		if err != nil {
+			logFatalln(err)
+		}
+	}
+
+	repoCmd.AddCommand(GetRepoCommand)
+}

cmd/datamon/cmd/repo_list.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"log"
-	"text/template"
 
 	"github.com/oneconcern/datamon/pkg/core"
 	"github.com/spf13/cobra"
@@ -15,9 +14,7 @@ var repoList = &cobra.Command{
 	Short: "List repos",
 	Long:  "List repos that have been created",
 	Run: func(cmd *cobra.Command, args []string) {
-		const listLineTemplateString = `{{.Name}} , {{.Description}} , {{with .Contributor}}{{.Name}} , {{.Email}}{{end}} , {{.Timestamp}}`
 		ctx := context.Background()
-		listLineTemplate := template.Must(template.New("list line").Parse(listLineTemplateString))
 		remoteStores, err := paramsToRemoteCmdStores(ctx, params)
 		if err != nil {
 			logFatalln(err)
@@ -28,7 +25,7 @@ var repoList = &cobra.Command{
 		}
 		for _, rd := range repos {
 			var buf bytes.Buffer
-			err := listLineTemplate.Execute(&buf, rd)
+			err := repoDescriptorTemplate.Execute(&buf, rd)
 			if err != nil {
 				log.Println("executing template:", err)
 			}

pkg/core/bundle.go

@@ -263,6 +263,11 @@ func PublishMetadata(ctx context.Context, bundle *Bundle) error {
 	return implPublishMetadata(ctx, bundle, true, defaultBundleEntriesPerFile)
 }
 
+// DownloadMetadata from the archive to main memory
+func DownloadMetadata(ctx context.Context, bundle *Bundle) error {
+	return implPublishMetadata(ctx, bundle, false, defaultBundleEntriesPerFile)
+}
+
 // implementation of PublishMetadata() with some additional parameters for test
 func implPublishMetadata(ctx context.Context, bundle *Bundle,
 	publish bool,

pkg/core/bundle_unpack.go

@@ -118,7 +118,16 @@ func unpackBundleDescriptor(ctx context.Context, bundle *Bundle, publish bool) e
 		}
 
 	case bundle.MetaStore != nil:
-		rdr, err = bundle.MetaStore.Get(ctx, model.GetArchivePathToBundle(bundle.RepoID, bundle.BundleID))
+		var has bool
+		archivePathToBundle := model.GetArchivePathToBundle(bundle.RepoID, bundle.BundleID)
+		has, err = bundle.MetaStore.Has(ctx, archivePathToBundle)
+		if err != nil {
+			return err
+		}
+		if !has {
+			return ErrNotFound
+		}
+		rdr, err = bundle.MetaStore.Get(ctx, archivePathToBundle)
 		if err != nil {
 			return err
 		}