Skip to content

Commit

Permalink
#461: add ability to save s3 remotes to config
Browse files Browse the repository at this point in the history
  • Loading branch information
Charlotte Godley committed Jun 20, 2018
1 parent 1d007dd commit ab153b7
Show file tree
Hide file tree
Showing 2 changed files with 85 additions and 38 deletions.
22 changes: 17 additions & 5 deletions cmd/dm/pkg/commands/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,36 +89,48 @@ func NewCmdS3(out io.Writer) *cobra.Command {
Long: "Online help: https://docs.dotmesh.com/references/cli/#list-remotes-dm-remote-v",
}
cmd.AddCommand(&cobra.Command{
Use: "add <remote-name> <key-id:secret-key>",
Use: "remote add <remote-name> <key-id:secret-key>",
Short: "Add an S3 remote",
Long: "Online help: https://docs.dotmesh.com/references/cli/#add-a-new-remote-dm-remote-add-name-user-hostname",

Run: func(cmd *cobra.Command, args []string) {
runHandlingError(func() error {
if len(args) != 2 {
if len(args) != 3 {
return fmt.Errorf(
"Please specify <remote-name> <key-id:secret-key>",
)
}
// remote := args[0]
awsCredentials := strings.SplitN(args[1], ":", 2)
remote := args[1]
awsCredentials := strings.SplitN(args[2], ":", 2)
if len(awsCredentials) != 2 {
return fmt.Errorf(
"Please specify key-id:secret-key, got %s", awsCredentials,
)
}
keyID := awsCredentials[0]
secretKey := awsCredentials[1]
sess := session.Must(session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(awsCredentials[0], awsCredentials[1], ""),
Credentials: credentials.NewStaticCredentials(keyID, secretKey, ""),
}))
// I don't think region actually matters, but if none is supplied the client complains
svc := s3.New(sess, aws.NewConfig().WithRegion("us-east-1"))
data, err := svc.ListBuckets(nil)
if err != nil {
return err
}
// TODO: should we error here? I just wanted to add something that would check we could talk to S3/AWS using the keys supplied
if len(data.Buckets) == 0 {
return fmt.Errorf("supplied credentials do not have any buckets available")
}
dm, err := remotes.NewDotmeshAPI(configPath)
if err != nil {
return err
}
err = dm.Configuration.AddS3Remote(remote, keyID, secretKey)
if err != nil {
return err
}
fmt.Fprintln(out, "s3 remote added.")
return nil
})
},
Expand Down
101 changes: 68 additions & 33 deletions cmd/dm/pkg/remotes/remotes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,15 @@ import (
"github.com/openzipkin/zipkin-go-opentracing/examples/middleware"
)

type Remote struct {
type Remote interface {
}

type S3Remote struct {
KeyID string
SecretKey string
}

type DMRemote struct {
User string
Hostname string
Port int `json:",omitempty"`
Expand All @@ -50,7 +58,7 @@ type Remote struct {
DefaultRemoteVolumes map[string]map[string]VolumeName
}

func (remote Remote) String() string {
func (remote DMRemote) String() string {
v := reflect.ValueOf(remote)
toString := ""
for i := 0; i < v.NumField(); i++ {
Expand All @@ -66,15 +74,17 @@ func (remote Remote) String() string {

type Configuration struct {
CurrentRemote string
Remotes map[string]*Remote
DMRemotes map[string]*DMRemote
S3Remotes map[string]*S3Remote
lock sync.Mutex
configPath string
}

func NewConfiguration(configPath string) (*Configuration, error) {
c := &Configuration{
configPath: configPath,
Remotes: make(map[string]*Remote),
DMRemotes: make(map[string]*DMRemote),
S3Remotes: make(map[string]*S3Remote),
}
if err := c.Load(); err != nil {
return nil, err
Expand Down Expand Up @@ -114,20 +124,20 @@ func (c *Configuration) save() error {
return nil
}

func (c *Configuration) GetRemote(name string) (*Remote, error) {
func (c *Configuration) GetRemote(name string) (*DMRemote, error) {
c.lock.Lock()
defer c.lock.Unlock()
r, ok := c.Remotes[name]
r, ok := c.DMRemotes[name]
if !ok {
return nil, fmt.Errorf("Unable to find remote '%s'", name)
}
return r, nil
}

func (c *Configuration) GetRemotes() map[string]*Remote {
func (c *Configuration) GetRemotes() map[string]*DMRemote {
c.lock.Lock()
defer c.lock.Unlock()
return c.Remotes
return c.DMRemotes
}

func (c *Configuration) GetCurrentRemote() string {
Expand All @@ -139,9 +149,13 @@ func (c *Configuration) GetCurrentRemote() string {
func (c *Configuration) SetCurrentRemote(remote string) error {
c.lock.Lock()
defer c.lock.Unlock()
_, ok := c.Remotes[remote]
_, ok := c.DMRemotes[remote]
if !ok {
return fmt.Errorf("No such remote '%s'", remote)
if _, ok = c.S3Remotes[remote]; ok {
return fmt.Errorf("Cannot switch to remote '%s' - is an S3 remote", remote)
} else {
return fmt.Errorf("No such remote '%s'", remote)
}
}
c.CurrentRemote = remote
return c.save()
Expand All @@ -154,7 +168,7 @@ func (c *Configuration) CurrentVolume() (string, error) {
}

func (c *Configuration) currentVolume() (string, error) {
r, ok := c.Remotes[c.CurrentRemote]
r, ok := c.DMRemotes[c.CurrentRemote]
if !ok {
return "", fmt.Errorf(
"Unable to find remote '%s', which was apparently current",
Expand All @@ -167,21 +181,21 @@ func (c *Configuration) currentVolume() (string, error) {
func (c *Configuration) SetCurrentVolume(volume string) error {
c.lock.Lock()
defer c.lock.Unlock()
_, ok := c.Remotes[c.CurrentRemote]
_, ok := c.DMRemotes[c.CurrentRemote]
if !ok {
return fmt.Errorf(
"Unable to find remote '%s', which was apparently current",
c.CurrentRemote,
)
}
(*c.Remotes[c.CurrentRemote]).CurrentVolume = volume
(*c.DMRemotes[c.CurrentRemote]).CurrentVolume = volume
return c.save()
}

func (c *Configuration) DefaultRemoteVolumeFor(peer, namespace, volume string) (string, string, bool) {
c.lock.Lock()
defer c.lock.Unlock()
defaultRemoteVolume, ok := c.Remotes[peer].DefaultRemoteVolumes[namespace][volume]
defaultRemoteVolume, ok := c.DMRemotes[peer].DefaultRemoteVolumes[namespace][volume]
if !ok {
return "", "", false
}
Expand All @@ -191,7 +205,7 @@ func (c *Configuration) DefaultRemoteVolumeFor(peer, namespace, volume string) (
func (c *Configuration) SetDefaultRemoteVolumeFor(peer, namespace, volume, remoteNamespace, remoteVolume string) error {
c.lock.Lock()
defer c.lock.Unlock()
remote, ok := c.Remotes[peer]
remote, ok := c.DMRemotes[peer]
if !ok {
return fmt.Errorf(
"Unable to find remote '%s'",
Expand All @@ -211,7 +225,7 @@ func (c *Configuration) SetDefaultRemoteVolumeFor(peer, namespace, volume, remot
func (c *Configuration) CurrentBranchFor(volume string) (string, error) {
c.lock.Lock()
defer c.lock.Unlock()
currentBranch, ok := c.Remotes[c.CurrentRemote].CurrentBranches[volume]
currentBranch, ok := c.DMRemotes[c.CurrentRemote].CurrentBranches[volume]
if !ok {
return DEFAULT_BRANCH, nil
}
Expand All @@ -235,27 +249,27 @@ func (c *Configuration) SetCurrentBranch(branch string) error {
if err != nil {
return err
}
c.Remotes[c.CurrentRemote].CurrentBranches[cur] = branch
c.DMRemotes[c.CurrentRemote].CurrentBranches[cur] = branch
return c.save()
}

func (c *Configuration) DeleteStateForVolume(volume string) error {
c.lock.Lock()
defer c.lock.Unlock()
_, ok := c.Remotes[c.CurrentRemote]
_, ok := c.DMRemotes[c.CurrentRemote]
if !ok {
return fmt.Errorf(
"Unable to find remote '%s', which was apparently current",
c.CurrentRemote,
)
}
delete(c.Remotes[c.CurrentRemote].CurrentBranches, volume)
if volume == c.Remotes[c.CurrentRemote].CurrentVolume {
c.Remotes[c.CurrentRemote].CurrentVolume = ""
delete(c.DMRemotes[c.CurrentRemote].CurrentBranches, volume)
if volume == c.DMRemotes[c.CurrentRemote].CurrentVolume {
c.DMRemotes[c.CurrentRemote].CurrentVolume = ""
}
n, v, err := ParseNamespacedVolume(volume)
if err == nil {
delete(c.Remotes[c.CurrentRemote].DefaultRemoteVolumes[n], v)
delete(c.DMRemotes[c.CurrentRemote].DefaultRemoteVolumes[n], v)
} else {
return err
}
Expand All @@ -265,31 +279,46 @@ func (c *Configuration) DeleteStateForVolume(volume string) error {
func (c *Configuration) SetCurrentBranchForVolume(volume, branch string) error {
c.lock.Lock()
defer c.lock.Unlock()
_, ok := c.Remotes[c.CurrentRemote]
_, ok := c.DMRemotes[c.CurrentRemote]
if !ok {
return fmt.Errorf(
"Unable to find remote '%s', which was apparently current",
c.CurrentRemote,
)
}
if c.Remotes[c.CurrentRemote].CurrentBranches == nil {
c.Remotes[c.CurrentRemote].CurrentBranches = map[string]string{}
if c.DMRemotes[c.CurrentRemote].CurrentBranches == nil {
c.DMRemotes[c.CurrentRemote].CurrentBranches = map[string]string{}
}
c.Remotes[c.CurrentRemote].CurrentBranches[volume] = branch
c.DMRemotes[c.CurrentRemote].CurrentBranches[volume] = branch
return c.save()
}

func (c *Configuration) RemoteExists(remote string) bool {
_, ok := c.Remotes[remote]
_, ok := c.DMRemotes[remote]
if !ok {
_, ok = c.S3Remotes[remote]
}
return ok
}

func (c *Configuration) AddS3Remote(remote, keyID, secretKey string) error {
ok := c.RemoteExists(remote)
if ok {
return fmt.Errorf("Remote exists '%s'", remote)
}
c.S3Remotes[remote] = &S3Remote{
KeyID: keyID,
SecretKey: secretKey,
}
return c.save()
}

func (c *Configuration) AddRemote(remote, user, hostname string, port int, apiKey string) error {
_, ok := c.Remotes[remote]
ok := c.RemoteExists(remote)
if ok {
return fmt.Errorf("Remote exists '%s'", remote)
}
c.Remotes[remote] = &Remote{
c.DMRemotes[remote] = &DMRemote{
User: user,
Hostname: hostname,
Port: port,
Expand All @@ -299,11 +328,17 @@ func (c *Configuration) AddRemote(remote, user, hostname string, port int, apiKe
}

func (c *Configuration) RemoveRemote(remote string) error {
_, ok := c.Remotes[remote]
_, ok := c.DMRemotes[remote]
if !ok {
return fmt.Errorf("No such remote '%s'", remote)
_, ok = c.S3Remotes[remote]
if ok {
delete(c.S3Remotes, remote)
} else {
return fmt.Errorf("No such remote '%s'", remote)
}
} else {
delete(c.DMRemotes, remote)
}
delete(c.Remotes, remote)
if c.CurrentRemote == remote {
c.CurrentRemote = ""
}
Expand All @@ -313,7 +348,7 @@ func (c *Configuration) RemoveRemote(remote string) error {
func (c *Configuration) ClusterFromRemote(remote string) (*JsonRpcClient, error) {
c.lock.Lock()
defer c.lock.Unlock()
remoteCreds, ok := c.Remotes[remote]
remoteCreds, ok := c.DMRemotes[remote]
if !ok {
return nil, fmt.Errorf("No such remote '%s'", remote)
}
Expand Down

0 comments on commit ab153b7

Please sign in to comment.