diff --git a/cmd/dm/pkg/commands/s3.go b/cmd/dm/pkg/commands/s3.go index 31f84788e..12aa19d11 100644 --- a/cmd/dm/pkg/commands/s3.go +++ b/cmd/dm/pkg/commands/s3.go @@ -21,6 +21,8 @@ import ( "github.com/spf13/cobra" ) +var cloneLocalVolume string + func NewCmdS3(out io.Writer) *cobra.Command { cmd := &cobra.Command{ Use: "s3", @@ -30,7 +32,8 @@ func NewCmdS3(out io.Writer) *cobra.Command { cmd.AddCommand(&cobra.Command{ Use: "remote add [@endpoint]", Short: "Add an S3 remote", - Long: "Online help: https://docs.dotmesh.com/references/cli/#add-a-new-remote-dm-remote-add-name-user-hostname", + // TODO update docs + Long: "Online help: https://docs.dotmesh.com/references/cli/#add-a-new-remote-dm-remote-add-name-user-hostname", Run: func(cmd *cobra.Command, args []string) { runHandlingError(func() error { @@ -86,5 +89,40 @@ func NewCmdS3(out io.Writer) *cobra.Command { }) }, }) + subCommand := &cobra.Command{ + Use: "clone-subset [--local-name=]", + Short: "Clone an s3 bucket, but only select a subset as dictated by prefixes. (for full bucket clones see dm clone as normal)", + // TODO add this to the docs + Long: "Online help: https://docs.dotmesh.com/references/cli/#add-a-new-remote-dm-remote-add-name-user-hostname", + + Run: func(cmd *cobra.Command, args []string) { + runHandlingError(func() error { + prefixes := strings.Split(args[3], ",") + dm, err := remotes.NewDotmeshAPI(configPath, verboseOutput) + if err != nil { + return err + } + transferId, err := dm.RequestS3SubsetTransfer( + args[0], + cloneLocalVolume, + args[2], + prefixes, + // TODO also switch to the remote? + ) + if err != nil { + return err + } + err = dm.PollTransfer(transferId, out) + if err != nil { + return err + } + + return nil + }) + }, + } + subCommand.PersistentFlags().StringVarP(&cloneLocalVolume, "local-name", "", "", + "Local dot name to create") + cmd.AddCommand(subCommand) return cmd } diff --git a/cmd/dm/pkg/remotes/api.go b/cmd/dm/pkg/remotes/api.go index 164154aa3..42308634a 100644 --- a/cmd/dm/pkg/remotes/api.go +++ b/cmd/dm/pkg/remotes/api.go @@ -888,6 +888,7 @@ type TransferRequest struct { type S3TransferRequest struct { KeyID string SecretKey string + Prefixes []string Endpoint string Direction string LocalNamespace string @@ -1087,6 +1088,54 @@ func (dm *DotmeshAPI) RequestTransfer( } +func (dm *DotmeshAPI) RequestS3SubTransfer( + direction, peer, + localFilesystemName, localBranchName, + remoteVolume string, + prefixes []string, +) (string, error) { + var transferId string + localNamespace, localVolume, err := ParseNamespacedVolume(localFilesystemName) + if err != nil { + return "", err + } + connectionInitiator := dm.Configuration.CurrentRemote + client, err := dm.Configuration.ClusterFromRemote(connectionInitiator, dm.verbose) + if err != nil { + return "", err + } + remote, err := dm.Configuration.GetRemote(peer) + if err != nil { + return "", err + } + s3Remote, ok := remote.(*S3Remote) + if ok { + + transferRequest := S3TransferRequest{ + KeyID: s3Remote.KeyID, + SecretKey: s3Remote.SecretKey, + Prefixes: prefixes, + Endpoint: s3Remote.Endpoint, + Direction: direction, + LocalNamespace: localNamespace, + LocalName: localVolume, + LocalBranchName: deMasterify(localBranchName), + RemoteName: remoteVolume, + // TODO add TargetSnapshot here, to support specifying "push to a given + // snapshot" rather than just "push all snapshots up to the latest" + } + + err = client.CallRemote(context.Background(), + "DotmeshRPC.S3Transfer", transferRequest, &transferId) + if err != nil { + return "", err + } + } else { + return "", fmt.Errorf("Unknown remote type %#v\n", remote) + } + return transferId, nil +} + // FIXME: Put this in a shared library, as it duplicates the copy in // dotmesh-server/pkg/main/utils.go (now with a few differences) diff --git a/cmd/dotmesh-server/types.go b/cmd/dotmesh-server/types.go index ea27f144a..8a5ae1182 100644 --- a/cmd/dotmesh-server/types.go +++ b/cmd/dotmesh-server/types.go @@ -199,6 +199,7 @@ type fsMachine struct { type S3TransferRequest struct { KeyID string SecretKey string + Prefixes []string Endpoint string Direction string LocalNamespace string