Merge pull request #160 from fairDataSociety/feature/configFile.1

server running from config values

cmd/dfs-cli/cmd/root.go

@@ -82,10 +82,6 @@ func init() {
 	defaultConfig := filepath.Join(home, ".fairOS/dfs-cli.yml")
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", defaultConfig, "config file")
 
-	// Cobra also supports local flags, which will only run
-	// when this action is called directly.
-	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
-
 	rootCmd.PersistentFlags().String("fdfsHost", "127.0.0.1", "fdfs host")
 	rootCmd.PersistentFlags().String("fdfsPort", "9090", "fdfs port")
 

cmd/dfs/cmd/config.go

@@ -0,0 +1,49 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
+)
+
+var (
+	optionCORSAllowedOrigins = "cors-allowed-origins"
+	optionDFSDataDir         = "dfs.data-dir"
+	optionDFSHttpPort        = "dfs.ports.http-port"
+	optionDFSPprofPort       = "dfs.ports.pprof-port"
+	optionVerbosity          = "verbosity"
+	optionBeeApi             = "bee.bee-api-endpoint"
+	optionBeeDebugApi        = "bee.bee-debug-api-endpoint"
+	optionBeePostageBatchId  = "bee.postage-batch-id"
+	optionCookieDomain       = "cookie-domain"
+
+	defaultCORSAllowedOrigins = []string{}
+	defaultDFSHttpPort        = ":9090"
+	defaultDFSPprofPort       = ":9091"
+	defaultVerbosity          = "trace"
+	defaultBeeApi             = "http://localhost:1633"
+	defaultBeeDebugApi        = "http://localhost:1635"
+	defaultCookieDomain       = "api.fairos.io"
+)
+
+var configCmd = &cobra.Command{
+	Use:   "config",
+	Short: "Print default or provided configuration in yaml format",
+	RunE: func(cmd *cobra.Command, args []string) (err error) {
+
+		if len(args) > 0 {
+			return cmd.Help()
+		}
+
+		d := config.AllSettings()
+		ym, err := yaml.Marshal(d)
+		if err != nil {
+			return err
+		}
+		cmd.Println(string(ym))
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(configCmd)
+}

cmd/dfs/cmd/root.go

@@ -21,30 +21,53 @@ import (
 	"os"
 	"path/filepath"
 
-	homedir "github.com/mitchellh/go-homedir"
+	"github.com/mitchellh/go-homedir"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
 var (
-	cfgFile   string
-	beeHost   string
-	beePort   string
-	verbosity string
-	dataDir   string
+	defaultDir    = filepath.Join(".fairOS", "dfs")
+	defaultConfig = ".dfs.yaml"
+
+	cfgFile     string
+	beeApi      string
+	beeDebugApi string
+	verbosity   string
+	dataDir     string
+
+	dataDirPath string
+	config      = viper.New()
 )
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
 	Use:   "dfs",
 	Short: "Decentralised file system over Swarm(https://ethswarm.org/)",
-	Long: `dfs is the file system layer of internetOS. It is a thin layer over Swarm.  
-It adds features to Swarm that is required by the internetOS to parallelize computation of data. 
+	Long: `dfs is the file system layer of fairOS. It is a thin layer over Swarm.  
+It adds features to Swarm that is required by the fairOS to parallelize computation of data. 
 It manages the metadata of directories and files created and expose them to higher layers.
 It can also be used as a standalone personal, decentralised drive over the internet`,
+	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		if err := config.BindPFlag(optionDFSDataDir, cmd.Flags().Lookup("dataDir")); err != nil {
+			return err
+		}
+		if err := config.BindPFlag(optionBeeApi, cmd.Flags().Lookup("beeApi")); err != nil {
+			return err
+		}
+		if err := config.BindPFlag(optionBeeDebugApi, cmd.Flags().Lookup("beeDebugApi")); err != nil {
+			return err
+		}
+		if err := config.BindPFlag(optionVerbosity, cmd.Flags().Lookup("verbosity")); err != nil {
+			return err
+		}
 
-	//Run: func(cmd *cobra.Command, args []string) {
-	//},
+		dataDir = config.GetString(optionDFSDataDir)
+		beeApi = config.GetString(optionBeeApi)
+		beeDebugApi = config.GetString(optionBeeDebugApi)
+		verbosity = config.GetString(optionVerbosity)
+		return nil
+	},
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.
@@ -76,46 +99,79 @@ func init() {
 		fmt.Println(err)
 		os.Exit(1)
 	}
-
 	// Here you will define your flags and configuration settings.
 	// Cobra supports persistent flags, which, if defined here,
 	// will be global for your application.
-	defaultConfig := filepath.Join(home, ".fairOS/dfs.yml")
-	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", defaultConfig, "config file")
-
-	// Cobra also supports local flags, which will only run
-	// when this action is called directly.
-	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
-
-	defaultDataDir := filepath.Join(home, ".fairOS/dfs")
-	rootCmd.PersistentFlags().StringVar(&dataDir, "dataDir", defaultDataDir, "store data in this dir")
-	rootCmd.PersistentFlags().StringVar(&beeHost, "beeHost", "127.0.0.1", "bee host")
-	rootCmd.PersistentFlags().StringVar(&beePort, "beePort", "1633", "bee port")
-	rootCmd.PersistentFlags().StringVar(&verbosity, "verbosity", "5", "verbosity level")
+	configPath := filepath.Join(home, defaultConfig)
+
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", configPath, "config file")
+
+	dataDirPath = filepath.Join(home, defaultDir)
+	rootCmd.PersistentFlags().String("dataDir", dataDirPath, "store data in this dir")
+	rootCmd.PersistentFlags().String("beeApi", "localhost:1633", "full bee api endpoint")
+	rootCmd.PersistentFlags().String("beeDebugApi", "localhost:1635", "full bee debug api endpoint")
+	rootCmd.PersistentFlags().String("verbosity", "trace", "verbosity level")
+
+	rootCmd.PersistentFlags().String("beeHost", "127.0.0.1", "bee host")
+	rootCmd.PersistentFlags().String("beePort", "1633", "bee port")
+	if err := rootCmd.PersistentFlags().MarkDeprecated("beeHost", "run --beeApi, full bee api endpoint"); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	if err := rootCmd.PersistentFlags().MarkDeprecated("beePort", "run --beeApi, full bee api endpoint"); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
 }
 
 // initConfig reads in config file and ENV variables if set.
 func initConfig() {
 	if cfgFile != "" {
+		// check file stat
+		if _, err := os.Stat(cfgFile); err != nil {
+			// if there is no configFile, write it
+			writeConfig()
+		}
 		// Use config file from the flag.
-		viper.SetConfigFile(cfgFile)
+		config.SetConfigFile(cfgFile)
 	} else {
 		// Find home dir.
 		home, err := homedir.Dir()
 		if err != nil {
 			fmt.Println(err)
 			os.Exit(1)
 		}
+		// check file stat
+		cfgFile = filepath.Join(home, defaultConfig)
+		if _, err := os.Stat(cfgFile); err != nil {
+			// if there is no configFile, write it
+			writeConfig()
+		}
 
-		// Search config in home dir with name ".dfs" (without extension).
-		viper.AddConfigPath(home)
-		viper.SetConfigName(".fairOS/dfs")
+		config.SetConfigFile(cfgFile)
 	}
 
-	viper.AutomaticEnv() // read in environment variables that match
-
+	config.AutomaticEnv() // read in environment variables that match
 	// If a config file is found, read it in.
-	if err := viper.ReadInConfig(); err == nil {
-		fmt.Println("Using config file:", viper.ConfigFileUsed())
+	if err := config.ReadInConfig(); err == nil {
+		fmt.Println("Using config file:", config.ConfigFileUsed())
+	}
+}
+
+func writeConfig() {
+	c := viper.New()
+	c.Set(optionCORSAllowedOrigins, defaultCORSAllowedOrigins)
+	c.Set(optionDFSDataDir, dataDirPath)
+	c.Set(optionDFSHttpPort, defaultDFSHttpPort)
+	c.Set(optionDFSPprofPort, defaultDFSPprofPort)
+	c.Set(optionVerbosity, defaultVerbosity)
+	c.Set(optionBeeApi, defaultBeeApi)
+	c.Set(optionBeeDebugApi, defaultBeeDebugApi)
+	c.Set(optionBeePostageBatchId, "")
+	c.Set(optionCookieDomain, defaultCookieDomain)
+
+	if err := c.WriteConfigAs(cfgFile); err != nil {
+		fmt.Println("failed to write config file")
+		os.Exit(1)
 	}
 }

cmd/dfs/cmd/server.go

@@ -46,7 +46,31 @@ var serverCmd = &cobra.Command{
 	Short: "starts a HTTP server for the dfs",
 	Long: `Serves all the dfs commands through an HTTP server so that the upper layers
 can consume it.`,
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		if err := config.BindPFlag(optionDFSHttpPort, cmd.Flags().Lookup("httpPort")); err != nil {
+			return err
+		}
+		if err := config.BindPFlag(optionDFSPprofPort, cmd.Flags().Lookup("pprofPort")); err != nil {
+			return err
+		}
+		if err := config.BindPFlag(optionCookieDomain, cmd.Flags().Lookup("cookieDomain")); err != nil {
+			return err
+		}
+		if err := config.BindPFlag(optionCORSAllowedOrigins, cmd.Flags().Lookup("cors-origins")); err != nil {
+			return err
+		}
+		if err := config.BindPFlag(optionBeePostageBatchId, cmd.Flags().Lookup("postageBlockId")); err != nil {
+			return err
+		}
+		return nil
+	},
 	Run: func(cmd *cobra.Command, args []string) {
+		httpPort = config.GetString(optionDFSHttpPort)
+		pprofPort = config.GetString(optionDFSPprofPort)
+		cookieDomain = config.GetString(optionCookieDomain)
+		postageBlockId = config.GetString(optionBeePostageBatchId)
+		corsOrigins = config.GetStringSlice(optionCORSAllowedOrigins)
+		verbosity = config.GetString(optionVerbosity)
 		if postageBlockId == "" {
 			_ = cmd.Help()
 			fmt.Println("\npostageBlockId is required to run server")
@@ -75,15 +99,15 @@ can consume it.`,
 		logger.Info("configuration values")
 		logger.Info("version        : ", dfs.Version)
 		logger.Info("dataDir        : ", dataDir)
-		logger.Info("beeHost        : ", beeHost)
-		logger.Info("beePort        : ", beePort)
+		logger.Info("beeApi         : ", beeApi)
+		logger.Info("beeDebugApi    : ", beeDebugApi)
 		logger.Info("verbosity      : ", verbosity)
 		logger.Info("httpPort       : ", httpPort)
 		logger.Info("pprofPort      : ", pprofPort)
 		logger.Info("cookieDomain   : ", cookieDomain)
 		logger.Info("postageBlockId : ", postageBlockId)
 		logger.Info("corsOrigins    : ", corsOrigins)
-		hdlr, err := api.NewHandler(dataDir, beeHost, beePort, cookieDomain, postageBlockId, logger)
+		hdlr, err := api.NewHandler(dataDir, beeApi, beeDebugApi, cookieDomain, postageBlockId, logger)
 		if err != nil {
 			logger.Error(err.Error())
 			return
@@ -94,11 +118,11 @@ can consume it.`,
 }
 
 func init() {
-	serverCmd.Flags().StringVar(&httpPort, "httpPort", "9090", "http port")
-	serverCmd.Flags().StringVar(&pprofPort, "pprofPort", "9091", "pprof port")
-	serverCmd.Flags().StringVar(&cookieDomain, "cookieDomain", "api.fairos.io", "the domain to use in the cookie")
-	serverCmd.Flags().StringVar(&postageBlockId, "postageBlockId", "", "the postage block used to store the data in bee")
-	serverCmd.Flags().StringSliceVar(&corsOrigins, "cors-origins", []string{}, "allow CORS headers for the given origins")
+	serverCmd.Flags().String("httpPort", defaultDFSHttpPort, "http port")
+	serverCmd.Flags().String("pprofPort", defaultDFSPprofPort, "pprof port")
+	serverCmd.Flags().String("cookieDomain", defaultCookieDomain, "the domain to use in the cookie")
+	serverCmd.Flags().String("postageBlockId", "", "the postage block used to store the data in bee")
+	serverCmd.Flags().StringSlice("cors-origins", defaultCORSAllowedOrigins, "allow CORS headers for the given origins")
 	rootCmd.AddCommand(serverCmd)
 }
 
@@ -122,7 +146,12 @@ func startHttpService(logger logging.Logger) {
 			logger.Errorf("error in API /: ", err)
 			return
 		}
-		_, err = fmt.Fprintln(w, beeHost+":"+beePort)
+		_, err = fmt.Fprintln(w, beeApi)
+		if err != nil {
+			logger.Errorf("error in API /: ", err)
+			return
+		}
+		_, err = fmt.Fprintln(w, beeDebugApi)
 		if err != nil {
 			logger.Errorf("error in API /: ", err)
 			return
@@ -274,15 +303,15 @@ func startHttpService(logger logging.Logger) {
 	// starting the pprof server
 	go func() {
 		logger.Infof("fairOS-dfs pprof listening on port: %v", pprofPort)
-		err := http.ListenAndServe("localhost:"+pprofPort, nil)
+		err := http.ListenAndServe("localhost"+pprofPort, nil)
 		if err != nil {
 			logger.Errorf("pprof listenAndServe: %v ", err.Error())
 			return
 		}
 	}()
 
 	logger.Infof("fairOS-dfs API server listening on port: %v", httpPort)
-	err := http.ListenAndServe(":"+httpPort, handler)
+	err := http.ListenAndServe(httpPort, handler)
 	if err != nil {
 		logger.Errorf("http listenAndServe: %v ", err.Error())
 		return

go.mod

@@ -28,5 +28,6 @@ require (
 	github.com/tyler-smith/go-bip39 v1.0.2
 	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
 	golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf
+	gopkg.in/yaml.v2 v2.3.0
 	resenje.org/jsonhttp v0.2.0
 )

pkg/api/handler.go

@@ -26,8 +26,8 @@ type Handler struct {
 	logger logging.Logger
 }
 
-func NewHandler(dataDir, beeHost, beePort, cookieDomain, postageBlockId string, logger logging.Logger) (*Handler, error) {
-	api, err := dfs.NewDfsAPI(dataDir, beeHost, beePort, cookieDomain, postageBlockId, logger)
+func NewHandler(dataDir, beeApi, beeDebugApi, cookieDomain, postageBlockId string, logger logging.Logger) (*Handler, error) {
+	api, err := dfs.NewDfsAPI(dataDir, beeApi, beeDebugApi, cookieDomain, postageBlockId, logger)
 	if err != nil {
 		return nil, dfs.ErrBeeClient
 	}

pkg/blockstore/bee/client.go

@@ -54,9 +54,8 @@ const (
 )
 
 type BeeClient struct {
-	host               string
-	port               string
 	url                string
+	debugUrl           string
 	client             *http.Client
 	hasher             *bmtlegacy.Hasher
 	chunkCache         *lru.Cache
@@ -75,7 +74,7 @@ type bytesPostResponse struct {
 }
 
 // NewBeeClient creates a new client which connects to the Swarm bee node to access the Swarm network.
-func NewBeeClient(host, port, postageBlockId string, logger logging.Logger) *BeeClient {
+func NewBeeClient(apiUrl, debugApiUrl, postageBlockId string, logger logging.Logger) *BeeClient {
 	p := bmtlegacy.NewTreePool(hashFunc, swarm.Branches, bmtlegacy.PoolSize)
 	cache, err := lru.New(chunkCacheSize)
 	if err != nil {
@@ -91,9 +90,8 @@ func NewBeeClient(host, port, postageBlockId string, logger logging.Logger) *Bee
 	}
 
 	return &BeeClient{
-		host:               host,
-		port:               port,
-		url:                fmt.Sprintf("http://" + host + ":" + port),
+		url:                apiUrl,
+		debugUrl:           debugApiUrl,
 		client:             createHTTPClient(),
 		hasher:             bmtlegacy.New(p),
 		chunkCache:         cache,
@@ -462,7 +460,7 @@ func (s *BeeClient) GetNewPostageBatch() error {
 	to := time.Now()
 	s.logger.Infof("Trying to get new postage batch id")
 	path := filepath.Join(postageBatchUrl, "10000000/20")
-	fullUrl := fmt.Sprintf(s.url + path)
+	fullUrl := fmt.Sprintf(s.debugUrl + path)
 	req, err := http.NewRequest(http.MethodPost, fullUrl, nil)
 	if err != nil {
 		return err

pkg/dfs/api.go

@@ -31,8 +31,8 @@ type DfsAPI struct {
 }
 
 // NewDfsAPI is the main entry point for the df controller.
-func NewDfsAPI(dataDir, host, port, cookieDomain, postageBlockId string, logger logging.Logger) (*DfsAPI, error) {
-	c := bee.NewBeeClient(host, port, postageBlockId, logger)
+func NewDfsAPI(dataDir, apiUrl, debugApiUrl, cookieDomain, postageBlockId string, logger logging.Logger) (*DfsAPI, error) {
+	c := bee.NewBeeClient(apiUrl, debugApiUrl, postageBlockId, logger)
 	if !c.CheckConnection() {
 		return nil, ErrBeeClient
 	}