#461: beginnings of client side code for subset clones

cmd/dm/pkg/commands/s3.go

@@ -21,6 +21,8 @@ import (
 	"github.com/spf13/cobra"
 )
 
+var cloneLocalVolume string
+
 func NewCmdS3(out io.Writer) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "s3",
@@ -30,7 +32,8 @@ func NewCmdS3(out io.Writer) *cobra.Command {
 	cmd.AddCommand(&cobra.Command{
 		Use:   "remote add <remote-name> <key-id:secret-key>[@endpoint]",
 		Short: "Add an S3 remote",
-		Long:  "Online help: https://docs.dotmesh.com/references/cli/#add-a-new-remote-dm-remote-add-name-user-hostname",
+		// TODO update docs
+		Long: "Online help: https://docs.dotmesh.com/references/cli/#add-a-new-remote-dm-remote-add-name-user-hostname",
 
 		Run: func(cmd *cobra.Command, args []string) {
 			runHandlingError(func() error {
@@ -86,5 +89,40 @@ func NewCmdS3(out io.Writer) *cobra.Command {
 			})
 		},
 	})
+	subCommand := &cobra.Command{
+		Use:   "clone-subset <remote> <bucket> <prefixes> [--local-name=<dot>]",
+		Short: "Clone an s3 bucket, but only select a subset as dictated by prefixes. (for full bucket clones see dm clone as normal)",
+		// TODO add this to the docs
+		Long: "Online help: https://docs.dotmesh.com/references/cli/#add-a-new-remote-dm-remote-add-name-user-hostname",
+
+		Run: func(cmd *cobra.Command, args []string) {
+			runHandlingError(func() error {
+				prefixes := strings.Split(args[3], ",")
+				dm, err := remotes.NewDotmeshAPI(configPath, verboseOutput)
+				if err != nil {
+					return err
+				}
+				transferId, err := dm.RequestS3SubsetTransfer(
+					args[0],
+					cloneLocalVolume,
+					args[2],
+					prefixes,
+					// TODO also switch to the remote?
+				)
+				if err != nil {
+					return err
+				}
+				err = dm.PollTransfer(transferId, out)
+				if err != nil {
+					return err
+				}
+
+				return nil
+			})
+		},
+	}
+	subCommand.PersistentFlags().StringVarP(&cloneLocalVolume, "local-name", "", "",
+		"Local dot name to create")
+	cmd.AddCommand(subCommand)
 	return cmd
 }

cmd/dm/pkg/remotes/api.go

@@ -888,6 +888,7 @@ type TransferRequest struct {
 type S3TransferRequest struct {
 	KeyID           string
 	SecretKey       string
+	Prefixes        []string
 	Endpoint        string
 	Direction       string
 	LocalNamespace  string
@@ -1087,6 +1088,54 @@ func (dm *DotmeshAPI) RequestTransfer(
 
 }
 
+func (dm *DotmeshAPI) RequestS3SubTransfer(
+	direction, peer,
+	localFilesystemName, localBranchName,
+	remoteVolume string,
+	prefixes []string,
+) (string, error) {
+	var transferId string
+	localNamespace, localVolume, err := ParseNamespacedVolume(localFilesystemName)
+	if err != nil {
+		return "", err
+	}
+	connectionInitiator := dm.Configuration.CurrentRemote
+	client, err := dm.Configuration.ClusterFromRemote(connectionInitiator, dm.verbose)
+	if err != nil {
+		return "", err
+	}
+	remote, err := dm.Configuration.GetRemote(peer)
+	if err != nil {
+		return "", err
+	}
+	s3Remote, ok := remote.(*S3Remote)
+	if ok {
+
+		transferRequest := S3TransferRequest{
+			KeyID:           s3Remote.KeyID,
+			SecretKey:       s3Remote.SecretKey,
+			Prefixes:        prefixes,
+			Endpoint:        s3Remote.Endpoint,
+			Direction:       direction,
+			LocalNamespace:  localNamespace,
+			LocalName:       localVolume,
+			LocalBranchName: deMasterify(localBranchName),
+			RemoteName:      remoteVolume,
+			// TODO add TargetSnapshot here, to support specifying "push to a given
+			// snapshot" rather than just "push all snapshots up to the latest"
+		}
+
+		err = client.CallRemote(context.Background(),
+			"DotmeshRPC.S3Transfer", transferRequest, &transferId)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		return "", fmt.Errorf("Unknown remote type %#v\n", remote)
+	}
+	return transferId, nil
+}
+
 // FIXME: Put this in a shared library, as it duplicates the copy in
 // dotmesh-server/pkg/main/utils.go (now with a few differences)
 

cmd/dotmesh-server/types.go

@@ -199,6 +199,7 @@ type fsMachine struct {
 type S3TransferRequest struct {
 	KeyID           string
 	SecretKey       string
+	Prefixes        []string
 	Endpoint        string
 	Direction       string
 	LocalNamespace  string