Repository Handling Updates: install and remove commands (#314)

* first draft of install command

* additional improvements to initialization for bundling compatibility

* remove unnecessary extra parameter in repo collection

* add appropriate absolute path handling to repo itself

* working repo removal command

* add test case to verify it all can work with no config file

* usage improvement, bugfix, add test case to ensure that everything works without config

* output formatting improvements and fixing pointer bugs

* appease pre-commit

* fix typo

* bugfix config compatibility with test cases

* fix spelling error

* remove underscores

* remove deprecated library usage

* remove additional underscore

cmd/README.md

@@ -29,10 +29,10 @@ This function is principally used for tests.
 
 ---
 
-### Execute(ExecOptions)
+### Execute()
 
 ```go
-Execute(ExecOptions) error
+Execute() error
 ```
 
 Execute sets up runtime configuration for the root command

cmd/config.go

@@ -21,44 +21,76 @@ package cmd
 
 import (
 	// 'go lint': need blank import for embedding default config
+	"bytes"
+	// needed for embedded filesystem
 	_ "embed"
+	"fmt"
 	"os"
 	"path/filepath"
 
+	"github.com/facebookincubator/ttpforge/pkg/logging"
+	"github.com/facebookincubator/ttpforge/pkg/repos"
 	"github.com/spf13/afero"
+	"gopkg.in/yaml.v3"
 )
 
+// Config stores the variables from the TTPForge global config file
+type Config struct {
+	RepoSpecs []repos.Spec `yaml:"repos"`
+
+	repoCollection repos.RepoCollection
+	cfgFile        string
+}
+
 var (
-	autoInitConfig bool
 	//go:embed default-config.yaml
 	defaultConfigContents string
 	defaultConfigFileName = "config.yaml"
 	defaultResourceDir    = ".ttpforge"
+
+	// Conf refers to the configuration used throughout TTPForge.
+	Conf = &Config{}
+
+	logConfig logging.Config
 )
 
-func ensureDefaultConfig() (string, error) {
+func getDefaultConfigFilePath() (string, error) {
 	homeDir, err := os.UserHomeDir()
 	if err != nil {
 		return "", err
 	}
+	defaultConfigPath := filepath.Join(homeDir, defaultResourceDir, defaultConfigFileName)
+	return defaultConfigPath, nil
+}
 
-	f := afero.NewOsFs()
-	defaultConfigDir := filepath.Join(homeDir, defaultResourceDir)
-	defaultConfigPath := filepath.Join(defaultConfigDir, defaultConfigFileName)
-	exists, err := afero.Exists(f, defaultConfigPath)
-	if err != nil {
-		return "", err
-	}
-	if exists {
-		return defaultConfigPath, nil
-	}
-
-	if err = os.MkdirAll(defaultConfigDir, 0700); err != nil {
-		return "", err
+// loadRepoCollection verifies that all repositories specified
+// in the configuration file are present on the filesystem
+// and clones missing ones if needed
+func (cfg *Config) loadRepoCollection() (repos.RepoCollection, error) {
+	// locate our config file directory to expend config-relative paths
+	var basePath string
+	if cfg.cfgFile != "" {
+		cfgFileAbsPath, err := filepath.Abs(cfg.cfgFile)
+		if err != nil {
+			return nil, err
+		}
+		basePath = filepath.Dir(cfgFileAbsPath)
 	}
+	fsys := afero.NewOsFs()
+	return repos.NewRepoCollection(fsys, cfg.RepoSpecs, basePath)
+}
 
-	if err := afero.WriteFile(f, defaultConfigPath, []byte(defaultConfigContents), 0600); err != nil {
-		return "", err
+// save() writes the current config back to its file - used by `install“ command
+func (cfg *Config) save() error {
+	var b bytes.Buffer
+	yamlEncoder := yaml.NewEncoder(&b)
+	yamlEncoder.SetIndent(2)
+	err := yamlEncoder.Encode(&cfg)
+	if err != nil {
+		return fmt.Errorf("marshalling config failed: %v", err)
 	}
-	return defaultConfigPath, nil
+	// YAML won't add this stylistic choice so we do it ourselves
+	cfgStr := "---\n" + b.String()
+	err = os.WriteFile(cfg.cfgFile, []byte(cfgStr), 0)
+	return err
 }

cmd/init.go

@@ -20,10 +20,24 @@ THE SOFTWARE.
 package cmd
 
 import (
+	"fmt"
+	"os"
+	"path/filepath"
+
 	"github.com/facebookincubator/ttpforge/pkg/logging"
+	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 )
 
+func copyEmbeddedConfigToPath(configFilePath string) error {
+	cfgDir := filepath.Dir(configFilePath)
+	if err := os.MkdirAll(cfgDir, 0700); err != nil {
+		return err
+	}
+	err := afero.WriteFile(afero.NewOsFs(), configFilePath, []byte(defaultConfigContents), 0600)
+	return err
+}
+
 func buildInitCommand() *cobra.Command {
 	return &cobra.Command{
 		Use:   "init",
@@ -33,7 +47,25 @@ TTPForge is a Purple Team engagement tool to execute Tactics, Techniques, and Pr
     `,
 		TraverseChildren: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			// initialization will be handled by config checking code
+			defaultConfigFilePath, err := getDefaultConfigFilePath()
+			if err != nil {
+				return fmt.Errorf("could not lookup default config file path: %v", err)
+			}
+			exists, err := afero.Exists(afero.NewOsFs(), defaultConfigFilePath)
+			if err != nil {
+				return fmt.Errorf("could not check existence of file %v: %v", defaultConfigFilePath, err)
+			}
+			if exists {
+				logging.L().Warnf("Configuration file %v already exists - TTPForge is probably already initialized", defaultConfigFilePath)
+				logging.L().Warn("If you really want to re-initialize it, delete all existing TTPForge configuration files/directories")
+				return nil
+			}
+
+			logging.L().Infof("Copying embedded configuration file to path: %v", defaultConfigFilePath)
+			err = copyEmbeddedConfigToPath(defaultConfigFilePath)
+			if err != nil {
+				return fmt.Errorf("could not copy default configuration to path %v: %v", defaultConfigFilePath, err)
+			}
 			logging.L().Infof("TTPForge Initialized. Now try `ttpforge run` :)")
 			return nil
 		},

cmd/install.go

@@ -0,0 +1,35 @@
+/*
+Copyright © 2023-present, Meta Platforms, Inc. and affiliates
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+func buildInstallCommand() *cobra.Command {
+	installCmd := &cobra.Command{
+		Use:              "install",
+		Short:            "install various types of resources used by TTPForge",
+		Long:             "For now, you just want to use the 'ttpforge install repo' subcommand",
+		TraverseChildren: true,
+	}
+	installCmd.AddCommand(buildInstallRepoCommand())
+	return installCmd
+}

cmd/installrepo.go

@@ -0,0 +1,67 @@
+/*
+Copyright © 2023-present, Meta Platforms, Inc. and affiliates
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+package cmd
+
+import (
+	"fmt"
+	"net/url"
+	"path/filepath"
+
+	"github.com/facebookincubator/ttpforge/pkg/logging"
+	"github.com/facebookincubator/ttpforge/pkg/repos"
+	"github.com/spf13/cobra"
+)
+
+func buildInstallRepoCommand() *cobra.Command {
+	var newRepoSpec repos.Spec
+	installRepoCommand := &cobra.Command{
+		Use:              "repo --name repo_name [repo_url]",
+		Short:            "install a new repository of TTPs for use by TTPForge",
+		TraverseChildren: true,
+		Args:             cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			u, err := url.ParseRequestURI(args[0])
+			if err != nil {
+				return fmt.Errorf("argument must be a valid URL - '%v' is not", args[0])
+			}
+			newRepoSpec.Git.URL = u.String()
+			newRepoSpec.Path = filepath.Join("repos", newRepoSpec.Name)
+			Conf.RepoSpecs = append(Conf.RepoSpecs, newRepoSpec)
+			_, err = Conf.loadRepoCollection()
+			if err != nil {
+				return fmt.Errorf("failed to add new repo: %v", err)
+			}
+
+			err = Conf.save()
+			if err != nil {
+				return fmt.Errorf("failed to save updated configuration: %v", err)
+			}
+			logging.L().Infof("New repository successfully installed!")
+			logging.L().Infof("Name: %v", newRepoSpec.Name)
+			logging.L().Infof("Path: %v", newRepoSpec.Path)
+			logging.L().Infof("List TTPs from your new repository with the command:")
+			logging.L().Infof("\tttpforge list ttps --repo %v", newRepoSpec.Name)
+			return nil
+		},
+	}
+	installRepoCommand.PersistentFlags().StringVar(&newRepoSpec.Name, "name", "", "The name to use for the new repository")
+	_ = installRepoCommand.MarkPersistentFlagRequired("name")
+	return installRepoCommand
+}

cmd/list_ttps.go → cmd/listttps.go

@@ -21,12 +21,14 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/spf13/cobra"
 )
 
 func buildListTTPsCommand() *cobra.Command {
-	return &cobra.Command{
+	var repoFilter string
+	listTTPsCommand := &cobra.Command{
 		Use:              "ttps",
 		Short:            "list TTPForge repos (in which TTPs live) that you have installed",
 		TraverseChildren: true,
@@ -36,10 +38,14 @@ func buildListTTPsCommand() *cobra.Command {
 				return err
 			}
 			for _, ttpRef := range ttpRefs {
+				if repoFilter != "" && !strings.HasPrefix(ttpRef, repoFilter+"//") {
+					continue
+				}
 				fmt.Println(ttpRef)
 			}
 			return nil
 		},
 	}
-
+	listTTPsCommand.PersistentFlags().StringVar(&repoFilter, "repo", "", "Show TTPs from only the specified repository")
+	return listTTPsCommand
 }

cmd/listttps_test.go

@@ -0,0 +1,54 @@
+/*
+Copyright © 2023-present, Meta Platforms, Inc. and affiliates
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+package cmd_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/facebookincubator/ttpforge/cmd"
+	"github.com/stretchr/testify/require"
+)
+
+func TestListTTPs(t *testing.T) {
+	testConfigFilePath := filepath.Join("test-resources", "test-config.yaml")
+	testCases := []struct {
+		name      string
+		ttpRef    string
+		wantError bool
+	}{
+		{
+			name: "no-filter",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			rc := cmd.BuildRootCommand()
+			rc.SetArgs([]string{"list", "ttps", "-c", testConfigFilePath})
+			err := rc.Execute()
+			if tc.wantError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}

cmd/remove.go

@@ -0,0 +1,35 @@
+/*
+Copyright © 2023-present, Meta Platforms, Inc. and affiliates
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+func buildRemoveCommand() *cobra.Command {
+	removeCmd := &cobra.Command{
+		Use:              "remove",
+		Short:            "remove (uninstall) various types of resources used by TTPForge",
+		Long:             "For now, you just want to use the 'ttpforge remove repo' subcommand",
+		TraverseChildren: true,
+	}
+	removeCmd.AddCommand(buildRemoveRepoCommand())
+	return removeCmd
+}