From f09d0b24c0e61be71518acdbeffc7f9d0271c545 Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Tue, 19 Dec 2023 15:18:26 -0600 Subject: [PATCH] :sparkles: Task report files (#577) Add support for: - TaskReport attached files. The addon (and tackle2-addon command) can attach files with command output and/or log files. - File API to support creating empty files. - File API to support append content - PATCH /files/:id. - Addon (adapter) to support attach files to the task report. Reaper updated to support fields with array of refs. New tag `ref=[]file` Example: ``` - '[CMD] Running: /usr/bin/konveyor-analyzer --provider-settings /addon/opt/settings.json --output-file /addon/report.yaml --no-dependency-rules --rules /addon/rules/rulesets/1/rules --rules /addon/rules/rulesets/24/rules --rules /addon/rules/rulesets/20/rules --label-selector konveyor.io/target=cloud-readiness --dep-label-selector !konveyor.io/dep-source=open-source' - '[CMD] /usr/bin/konveyor-analyzer succeeded.' - '[CMD] Running: /usr/bin/konveyor-analyzer-dep --provider-settings /addon/opt/settings.json --output-file /addon/deps.yaml' - '[CMD] /usr/bin/konveyor-analyzer-dep succeeded.' - 'Analysis reported. duration: 102.5784ms' - '[TAG] Tagging Application 4.' - Facts updated. - Done. attached: - id: 989 name: konveyor-analyzer.output activity: 44 - id: 991 name: konveyor-analyzer-dep.output activity: 50 ``` --------- Signed-off-by: Jeff Ortel --- addon/task.go | 38 +++ api/file.go | 60 +++++ api/task.go | 126 ++++++++-- binding/client.go | 46 +++- binding/file.go | 26 ++ migration/pkg.go | 2 + migration/v12/migrate.go | 20 ++ migration/v12/model/analysis.go | 166 +++++++++++++ migration/v12/model/application.go | 303 +++++++++++++++++++++++ migration/v12/model/assessment.go | 47 ++++ migration/v12/model/core.go | 379 +++++++++++++++++++++++++++++ migration/v12/model/pkg.go | 55 +++++ model/pkg.go | 2 +- reaper/file.go | 1 + reaper/ref.go | 21 ++ test/api/file/api_test.go | 66 +++++ 16 files changed, 1324 insertions(+), 34 deletions(-) create mode 100644 migration/v12/migrate.go create mode 100644 migration/v12/model/analysis.go create mode 100644 migration/v12/model/application.go create mode 100644 migration/v12/model/assessment.go create mode 100644 migration/v12/model/core.go create mode 100644 migration/v12/model/pkg.go diff --git a/addon/task.go b/addon/task.go index bfc3cca8..99e17f44 100644 --- a/addon/task.go +++ b/addon/task.go @@ -151,6 +151,44 @@ func (h *Task) Activity(entry string, v ...interface{}) { return } +// +// Attach ensures the file is attached to the report +// associated with the last entry in the activity. +func (h *Task) Attach(f *api.File) { + index := len(h.report.Activity) + h.AttachAt(f, index) + return +} + +// +// AttachAt ensures the file is attached to +// the report indexed to the activity. +// The activity is a 1-based index. Zero(0) means NOT associated. +func (h *Task) AttachAt(f *api.File, activity int) { + for _, ref := range h.report.Attached { + if ref.ID == f.ID { + return + } + } + Log.Info( + "Addon attached: %s", + "path", + f.Path, + "activity", + activity) + h.report.Attached = append( + h.report.Attached, + api.Attachment{ + Activity: activity, + Ref: api.Ref{ + ID: f.ID, + Name: f.Name, + }, + }) + h.pushReport() + return +} + // // Total report addon total items. func (h *Task) Total(n int) { diff --git a/api/file.go b/api/file.go index 932b892f..6f907f79 100644 --- a/api/file.go +++ b/api/file.go @@ -33,6 +33,7 @@ func (h FileHandler) AddRoutes(e *gin.Engine) { routeGroup.GET(FilesRoot+"/", h.List) routeGroup.POST(FileRoot, h.Create) routeGroup.PUT(FileRoot, h.Create) + routeGroup.PATCH(FileRoot, h.Append) routeGroup.GET(FileRoot, h.Get) routeGroup.DELETE(FileRoot, h.Delete) } @@ -122,6 +123,65 @@ func (h FileHandler) Create(ctx *gin.Context) { h.Respond(ctx, http.StatusCreated, r) } +// Append godoc +// @summary Append a file. +// @description Append a file. +// @tags file +// @accept json +// @produce json +// @success 204 +// @router /files/{id} [put] +// @param name id uint true "File ID" +func (h FileHandler) Append(ctx *gin.Context) { + var err error + input, err := ctx.FormFile(FileField) + if err != nil { + err = &BadRequestError{err.Error()} + _ = ctx.Error(err) + return + } + m := &model.File{} + id := h.pk(ctx) + db := h.DB(ctx) + err = db.First(m, id).Error + if err != nil { + _ = ctx.Error(err) + return + } + reader, err := input.Open() + if err != nil { + err = &BadRequestError{err.Error()} + _ = ctx.Error(err) + return + } + defer func() { + _ = reader.Close() + }() + writer, err := os.OpenFile(m.Path, os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + _ = ctx.Error(err) + return + } + defer func() { + _ = writer.Close() + }() + _, err = io.Copy(writer, reader) + if err != nil { + _ = ctx.Error(err) + return + } + db = h.DB(ctx) + db = db.Model(m) + user := h.BaseHandler.CurrentUser(ctx) + err = db.Update("UpdateUser", user).Error + if err != nil { + _ = ctx.Error(err) + return + } + + h.Status(ctx, http.StatusNoContent) +} + // Get godoc // @summary Get a file by ID. // @description Get a file by ID. Returns api.File when Accept=application/json else the file content. diff --git a/api/task.go b/api/task.go index 18aca35f..ea986f55 100644 --- a/api/task.go +++ b/api/task.go @@ -7,8 +7,13 @@ import ( tasking "github.com/konveyor/tackle2-hub/task" "gorm.io/gorm" "gorm.io/gorm/clause" + "io/ioutil" k8serr "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/utils/strings/slices" "net/http" + "sort" + "strconv" + "strings" "time" ) @@ -83,6 +88,14 @@ func (h TaskHandler) Get(ctx *gin.Context) { } r := Task{} r.With(task) + q := ctx.Query("merged") + if b, _ := strconv.ParseBool(q); b { + err := r.injectFiles(h.DB(ctx)) + if err != nil { + _ = ctx.Error(result.Error) + return + } + } h.Respond(ctx, http.StatusOK, r) } @@ -510,26 +523,27 @@ type TaskError struct { // Task REST resource. type Task struct { Resource `yaml:",inline"` - Name string `json:"name"` - Locator string `json:"locator,omitempty" yaml:",omitempty"` - Priority int `json:"priority,omitempty" yaml:",omitempty"` - Variant string `json:"variant,omitempty" yaml:",omitempty"` - Policy string `json:"policy,omitempty" yaml:",omitempty"` - TTL *TTL `json:"ttl,omitempty" yaml:",omitempty"` - Addon string `json:"addon,omitempty" binding:"required" yaml:",omitempty"` - Data interface{} `json:"data" swaggertype:"object" binding:"required"` - Application *Ref `json:"application,omitempty" yaml:",omitempty"` - State string `json:"state"` - Image string `json:"image,omitempty" yaml:",omitempty"` - Pod string `json:"pod,omitempty" yaml:",omitempty"` - Retries int `json:"retries,omitempty" yaml:",omitempty"` - Started *time.Time `json:"started,omitempty" yaml:",omitempty"` - Terminated *time.Time `json:"terminated,omitempty" yaml:",omitempty"` - Canceled bool `json:"canceled,omitempty" yaml:",omitempty"` - Bucket *Ref `json:"bucket,omitempty" yaml:",omitempty"` - Purged bool `json:"purged,omitempty" yaml:",omitempty"` - Errors []TaskError `json:"errors,omitempty" yaml:",omitempty"` - Activity []string `json:"activity,omitempty" yaml:",omitempty"` + Name string `json:"name"` + Locator string `json:"locator,omitempty" yaml:",omitempty"` + Priority int `json:"priority,omitempty" yaml:",omitempty"` + Variant string `json:"variant,omitempty" yaml:",omitempty"` + Policy string `json:"policy,omitempty" yaml:",omitempty"` + TTL *TTL `json:"ttl,omitempty" yaml:",omitempty"` + Addon string `json:"addon,omitempty" binding:"required" yaml:",omitempty"` + Data interface{} `json:"data" swaggertype:"object" binding:"required"` + Application *Ref `json:"application,omitempty" yaml:",omitempty"` + State string `json:"state"` + Image string `json:"image,omitempty" yaml:",omitempty"` + Pod string `json:"pod,omitempty" yaml:",omitempty"` + Retries int `json:"retries,omitempty" yaml:",omitempty"` + Started *time.Time `json:"started,omitempty" yaml:",omitempty"` + Terminated *time.Time `json:"terminated,omitempty" yaml:",omitempty"` + Canceled bool `json:"canceled,omitempty" yaml:",omitempty"` + Bucket *Ref `json:"bucket,omitempty" yaml:",omitempty"` + Purged bool `json:"purged,omitempty" yaml:",omitempty"` + Errors []TaskError `json:"errors,omitempty" yaml:",omitempty"` + Activity []string `json:"activity,omitempty" yaml:",omitempty"` + Attached []Attachment `json:"attached" yaml:",omitempty"` } // @@ -563,6 +577,7 @@ func (r *Task) With(m *model.Task) { report.With(m.Report) r.Activity = report.Activity r.Errors = append(report.Errors, r.Errors...) + r.Attached = report.Attached switch r.State { case tasking.Succeeded: switch report.Status { @@ -594,17 +609,58 @@ func (r *Task) Model() (m *model.Task) { return } +// +// injectFiles inject attached files into the activity. +func (r *Task) injectFiles(db *gorm.DB) (err error) { + sort.Slice( + r.Attached, + func(i, j int) bool { + return r.Attached[i].Activity > r.Attached[j].Activity + }) + for _, ref := range r.Attached { + if ref.Activity == 0 { + continue + } + if ref.Activity > len(r.Activity) { + continue + } + m := &model.File{} + err = db.First(m, ref.ID).Error + if err != nil { + return + } + b, nErr := ioutil.ReadFile(m.Path) + if nErr != nil { + err = nErr + return + } + var content []string + for _, s := range strings.Split(string(b), "\n") { + content = append( + content, + "> "+s) + } + snipA := slices.Clone(r.Activity[:ref.Activity]) + snipB := slices.Clone(r.Activity[ref.Activity:]) + r.Activity = append( + append(snipA, content...), + snipB...) + } + return +} + // // TaskReport REST resource. type TaskReport struct { Resource `yaml:",inline"` - Status string `json:"status"` - Errors []TaskError `json:"errors,omitempty" yaml:",omitempty"` - Total int `json:"total,omitempty" yaml:",omitempty"` - Completed int `json:"completed,omitempty" yaml:",omitempty"` - Activity []string `json:"activity,omitempty" yaml:",omitempty"` - Result interface{} `json:"result,omitempty" yaml:",omitempty" swaggertype:"object"` - TaskID uint `json:"task"` + Status string `json:"status"` + Errors []TaskError `json:"errors,omitempty" yaml:",omitempty"` + Total int `json:"total,omitempty" yaml:",omitempty"` + Completed int `json:"completed,omitempty" yaml:",omitempty"` + Activity []string `json:"activity,omitempty" yaml:",omitempty"` + Attached []Attachment `json:"attached,omitempty" yaml:",omitempty"` + Result interface{} `json:"result,omitempty" yaml:",omitempty" swaggertype:"object"` + TaskID uint `json:"task"` } // @@ -621,6 +677,9 @@ func (r *TaskReport) With(m *model.TaskReport) { if m.Errors != nil { _ = json.Unmarshal(m.Errors, &r.Errors) } + if m.Attached != nil { + _ = json.Unmarshal(m.Attached, &r.Attached) + } if m.Result != nil { _ = json.Unmarshal(m.Result, &r.Result) } @@ -647,7 +706,20 @@ func (r *TaskReport) Model() (m *model.TaskReport) { if r.Errors != nil { m.Errors, _ = json.Marshal(r.Errors) } + if r.Attached != nil { + m.Attached, _ = json.Marshal(r.Attached) + } m.ID = r.ID return } + +// +// Attachment associates Files with a TaskReport. +type Attachment struct { + // Ref references an attached File. + Ref `yaml:",inline"` + // Activity index (1-based) association with an + // activity entry. Zero(0) indicates not associated. + Activity int `json:"activity,omitempty" yaml:",omitempty"` +} diff --git a/binding/client.go b/binding/client.go index 40940a91..0dbf7278 100644 --- a/binding/client.go +++ b/binding/client.go @@ -422,9 +422,19 @@ func (r *Client) FileGet(path, destination string) (err error) { } // -// FilePut uploads a file. +// FilePost uploads a file. // Returns the created File resource. -func (r *Client) FilePut(path, source string, object interface{}) (err error) { +func (r *Client) FilePost(path, source string, object interface{}) (err error) { + if source == "" { + fields := []Field{ + { + Name: api.FileField, + Reader: bytes.NewReader([]byte{}), + }, + } + err = r.FileSend(path, http.MethodPost, fields, object) + return + } isDir, nErr := r.IsDir(source, true) if nErr != nil { err = nErr @@ -440,14 +450,24 @@ func (r *Client) FilePut(path, source string, object interface{}) (err error) { Path: source, }, } - err = r.FileSend(path, http.MethodPut, fields, object) + err = r.FileSend(path, http.MethodPost, fields, object) return } // -// FilePost uploads a file. +// FilePut uploads a file. // Returns the created File resource. -func (r *Client) FilePost(path, source string, object interface{}) (err error) { +func (r *Client) FilePut(path, source string, object interface{}) (err error) { + if source == "" { + fields := []Field{ + { + Name: api.FileField, + Reader: bytes.NewReader([]byte{}), + }, + } + err = r.FileSend(path, http.MethodPut, fields, object) + return + } isDir, nErr := r.IsDir(source, true) if nErr != nil { err = nErr @@ -463,7 +483,21 @@ func (r *Client) FilePost(path, source string, object interface{}) (err error) { Path: source, }, } - err = r.FileSend(path, http.MethodPost, fields, object) + err = r.FileSend(path, http.MethodPut, fields, object) + return +} + +// +// FilePatch appends file. +// Returns the created File resource. +func (r *Client) FilePatch(path string, buffer []byte) (err error) { + fields := []Field{ + { + Name: api.FileField, + Reader: bytes.NewReader(buffer), + }, + } + err = r.FileSend(path, http.MethodPatch, fields, nil) return } diff --git a/binding/file.go b/binding/file.go index 93f21f6c..6e9f258f 100644 --- a/binding/file.go +++ b/binding/file.go @@ -33,6 +33,24 @@ func (h *File) Get(id uint, destination string) (err error) { return } +// +// Touch creates an empty file. +func (h *File) Touch(name string) (r *api.File, err error) { + r = &api.File{} + path := Path(api.FileRoot).Inject(Params{api.ID: name}) + err = h.client.FilePost(path, "", r) + return +} + +// +// Post uploads a file. +func (h *File) Post(source string) (r *api.File, err error) { + r = &api.File{} + path := Path(api.FileRoot).Inject(Params{api.ID: pathlib.Base(source)}) + err = h.client.FilePost(path, source, r) + return +} + // // Put uploads a file. func (h *File) Put(source string) (r *api.File, err error) { @@ -42,6 +60,14 @@ func (h *File) Put(source string) (r *api.File, err error) { return } +// +// Patch appends a file. +func (h *File) Patch(id uint, buffer []byte) (err error) { + path := Path(api.FileRoot).Inject(Params{api.ID: id}) + err = h.client.FilePatch(path, buffer) + return +} + // // Delete a file. func (h *File) Delete(id uint) (err error) { diff --git a/migration/pkg.go b/migration/pkg.go index c73be645..61a0c99d 100644 --- a/migration/pkg.go +++ b/migration/pkg.go @@ -4,6 +4,7 @@ import ( "github.com/jortel/go-utils/logr" v10 "github.com/konveyor/tackle2-hub/migration/v10" v11 "github.com/konveyor/tackle2-hub/migration/v11" + v12 "github.com/konveyor/tackle2-hub/migration/v12" "github.com/konveyor/tackle2-hub/migration/v2" v3 "github.com/konveyor/tackle2-hub/migration/v3" v4 "github.com/konveyor/tackle2-hub/migration/v4" @@ -55,5 +56,6 @@ func All() []Migration { v9.Migration{}, v10.Migration{}, v11.Migration{}, + v12.Migration{}, } } diff --git a/migration/v12/migrate.go b/migration/v12/migrate.go new file mode 100644 index 00000000..dc991213 --- /dev/null +++ b/migration/v12/migrate.go @@ -0,0 +1,20 @@ +package v12 + +import ( + "github.com/jortel/go-utils/logr" + "github.com/konveyor/tackle2-hub/migration/v12/model" + "gorm.io/gorm" +) + +var log = logr.WithName("migration|v12") + +type Migration struct{} + +func (r Migration) Apply(db *gorm.DB) (err error) { + err = db.AutoMigrate(r.Models()...) + return +} + +func (r Migration) Models() []interface{} { + return model.All() +} diff --git a/migration/v12/model/analysis.go b/migration/v12/model/analysis.go new file mode 100644 index 00000000..e7e297af --- /dev/null +++ b/migration/v12/model/analysis.go @@ -0,0 +1,166 @@ +package model + +import "gorm.io/gorm" + +// +// Analysis report. +type Analysis struct { + Model + Effort int + Archived bool `json:"archived"` + Summary JSON `gorm:"type:json"` + Issues []Issue `gorm:"constraint:OnDelete:CASCADE"` + Dependencies []TechDependency `gorm:"constraint:OnDelete:CASCADE"` + ApplicationID uint `gorm:"index;not null"` + Application *Application +} + +// +// TechDependency report dependency. +type TechDependency struct { + Model + Provider string `gorm:"uniqueIndex:depA"` + Name string `gorm:"uniqueIndex:depA"` + Version string `gorm:"uniqueIndex:depA"` + SHA string `gorm:"uniqueIndex:depA"` + Indirect bool + Labels JSON `gorm:"type:json"` + AnalysisID uint `gorm:"index;uniqueIndex:depA;not null"` + Analysis *Analysis +} + +// +// Issue report issue (violation). +type Issue struct { + Model + RuleSet string `gorm:"uniqueIndex:issueA;not null"` + Rule string `gorm:"uniqueIndex:issueA;not null"` + Name string `gorm:"index"` + Description string + Category string `gorm:"index;not null"` + Incidents []Incident `gorm:"foreignKey:IssueID;constraint:OnDelete:CASCADE"` + Links JSON `gorm:"type:json"` + Facts JSON `gorm:"type:json"` + Labels JSON `gorm:"type:json"` + Effort int `gorm:"index;not null"` + AnalysisID uint `gorm:"index;uniqueIndex:issueA;not null"` + Analysis *Analysis +} + +// +// Incident report an issue incident. +type Incident struct { + Model + File string `gorm:"index;not null"` + Line int + Message string + CodeSnip string + Facts JSON `gorm:"type:json"` + IssueID uint `gorm:"index;not null"` + Issue *Issue +} + +// +// Link URL link. +type Link struct { + URL string `json:"url"` + Title string `json:"title,omitempty"` +} + +// +// ArchivedIssue resource created when issues are archived. +type ArchivedIssue struct { + RuleSet string `json:"ruleSet"` + Rule string `json:"rule"` + Name string `json:"name,omitempty" yaml:",omitempty"` + Description string `json:"description,omitempty" yaml:",omitempty"` + Category string `json:"category"` + Effort int `json:"effort"` + Incidents int `json:"incidents"` +} + +// +// RuleSet - Analysis ruleset. +type RuleSet struct { + Model + UUID *string `gorm:"uniqueIndex"` + Kind string + Name string `gorm:"uniqueIndex;not null"` + Description string + Repository JSON `gorm:"type:json"` + IdentityID *uint `gorm:"index"` + Identity *Identity + Rules []Rule `gorm:"constraint:OnDelete:CASCADE"` + DependsOn []RuleSet `gorm:"many2many:RuleSetDependencies;constraint:OnDelete:CASCADE"` +} + +func (r *RuleSet) Builtin() bool { + return r.UUID != nil +} + +// +// BeforeUpdate hook to avoid cyclic dependencies. +func (r *RuleSet) BeforeUpdate(db *gorm.DB) (err error) { + seen := make(map[uint]bool) + var nextDeps []RuleSet + var nextRuleSetIDs []uint + for _, dep := range r.DependsOn { + nextRuleSetIDs = append(nextRuleSetIDs, dep.ID) + } + for len(nextRuleSetIDs) != 0 { + result := db.Preload("DependsOn").Where("ID IN ?", nextRuleSetIDs).Find(&nextDeps) + if result.Error != nil { + err = result.Error + return + } + nextRuleSetIDs = nextRuleSetIDs[:0] + for _, nextDep := range nextDeps { + for _, dep := range nextDep.DependsOn { + if seen[dep.ID] { + continue + } + if dep.ID == r.ID { + err = DependencyCyclicError{} + return + } + seen[dep.ID] = true + nextRuleSetIDs = append(nextRuleSetIDs, dep.ID) + } + } + } + + return +} + +// +// Rule - Analysis rule. +type Rule struct { + Model + Name string + Description string + Labels JSON `gorm:"type:json"` + RuleSetID uint `gorm:"uniqueIndex:RuleA;not null"` + RuleSet *RuleSet + FileID *uint `gorm:"uniqueIndex:RuleA" ref:"file"` + File *File +} + +// +// Target - analysis rule selector. +type Target struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"uniqueIndex;not null"` + Description string + Provider string + Choice bool + Labels JSON `gorm:"type:json"` + ImageID uint `gorm:"index" ref:"file"` + Image *File + RuleSetID *uint `gorm:"index"` + RuleSet *RuleSet +} + +func (r *Target) Builtin() bool { + return r.UUID != nil +} diff --git a/migration/v12/model/application.go b/migration/v12/model/application.go new file mode 100644 index 00000000..f6c1586c --- /dev/null +++ b/migration/v12/model/application.go @@ -0,0 +1,303 @@ +package model + +import ( + "fmt" + "gorm.io/gorm" + "sync" + "time" +) + +type Application struct { + Model + BucketOwner + Name string `gorm:"index;unique;not null"` + Description string + Review *Review `gorm:"constraint:OnDelete:CASCADE"` + Repository JSON `gorm:"type:json"` + Binary string + Facts []Fact `gorm:"constraint:OnDelete:CASCADE"` + Comments string + Tasks []Task `gorm:"constraint:OnDelete:CASCADE"` + Tags []Tag `gorm:"many2many:ApplicationTags"` + Identities []Identity `gorm:"many2many:ApplicationIdentity;constraint:OnDelete:CASCADE"` + BusinessServiceID *uint `gorm:"index"` + BusinessService *BusinessService + OwnerID *uint `gorm:"index"` + Owner *Stakeholder `gorm:"foreignKey:OwnerID"` + Contributors []Stakeholder `gorm:"many2many:ApplicationContributors;constraint:OnDelete:CASCADE"` + Analyses []Analysis `gorm:"constraint:OnDelete:CASCADE"` + MigrationWaveID *uint `gorm:"index"` + MigrationWave *MigrationWave + Ticket *Ticket `gorm:"constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"constraint:OnDelete:CASCADE"` +} + +type Fact struct { + ApplicationID uint `gorm:"<-:create;primaryKey"` + Key string `gorm:"<-:create;primaryKey"` + Source string `gorm:"<-:create;primaryKey;not null"` + Value JSON `gorm:"type:json;not null"` + Application *Application +} + +// +// ApplicationTag represents a row in the join table for the +// many-to-many relationship between Applications and Tags. +type ApplicationTag struct { + ApplicationID uint `gorm:"primaryKey"` + TagID uint `gorm:"primaryKey"` + Source string `gorm:"primaryKey;not null"` + Application Application `gorm:"constraint:OnDelete:CASCADE"` + Tag Tag `gorm:"constraint:OnDelete:CASCADE"` +} + +// +// TableName must return "ApplicationTags" to ensure compatibility +// with the autogenerated join table name. +func (ApplicationTag) TableName() string { + return "ApplicationTags" +} + +// +// depMutex ensures Dependency.Create() is not executed concurrently. +var depMutex sync.Mutex + +type Dependency struct { + Model + ToID uint `gorm:"index"` + To *Application `gorm:"foreignKey:ToID;constraint:OnDelete:CASCADE"` + FromID uint `gorm:"index"` + From *Application `gorm:"foreignKey:FromID;constraint:OnDelete:CASCADE"` +} + +// +// Create a dependency synchronized using a mutex. +func (r *Dependency) Create(db *gorm.DB) (err error) { + depMutex.Lock() + defer depMutex.Unlock() + err = db.Create(r).Error + return +} + +// +// Validation Hook to avoid cyclic dependencies. +func (r *Dependency) BeforeCreate(db *gorm.DB) (err error) { + var nextDeps []*Dependency + var nextAppsIDs []uint + nextAppsIDs = append(nextAppsIDs, r.FromID) + for len(nextAppsIDs) != 0 { + db.Where("ToID IN ?", nextAppsIDs).Find(&nextDeps) + nextAppsIDs = nextAppsIDs[:0] // empty array, but keep capacity + for _, nextDep := range nextDeps { + if nextDep.FromID == r.ToID { + err = DependencyCyclicError{} + return + } + nextAppsIDs = append(nextAppsIDs, nextDep.FromID) + } + } + + return +} + +// +// Custom error type to allow API recognize Cyclic Dependency error and assign proper status code. +type DependencyCyclicError struct{} + +func (err DependencyCyclicError) Error() string { + return "cyclic dependencies are not allowed" +} + +type BusinessService struct { + Model + Name string `gorm:"index;unique;not null"` + Description string + Applications []Application `gorm:"constraint:OnDelete:SET NULL"` + StakeholderID *uint `gorm:"index"` + Stakeholder *Stakeholder +} + +type JobFunction struct { + Model + UUID *string `gorm:"uniqueIndex"` + Username string + Name string `gorm:"index;unique;not null"` + Stakeholders []Stakeholder `gorm:"constraint:OnDelete:SET NULL"` +} + +type Stakeholder struct { + Model + Name string `gorm:"not null;"` + Email string `gorm:"index;unique;not null"` + Groups []StakeholderGroup `gorm:"many2many:StakeholderGroupStakeholder;constraint:OnDelete:CASCADE"` + BusinessServices []BusinessService `gorm:"constraint:OnDelete:SET NULL"` + JobFunctionID *uint `gorm:"index"` + JobFunction *JobFunction + Owns []Application `gorm:"foreignKey:OwnerID;constraint:OnDelete:SET NULL"` + Contributes []Application `gorm:"many2many:ApplicationContributors;constraint:OnDelete:CASCADE"` + MigrationWaves []MigrationWave `gorm:"many2many:MigrationWaveStakeholders;constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"many2many:AssessmentStakeholders;constraint:OnDelete:CASCADE"` + Archetypes []Archetype `gorm:"many2many:ArchetypeStakeholders;constraint:OnDelete:CASCADE"` +} + +type StakeholderGroup struct { + Model + Name string `gorm:"index;unique;not null"` + Username string + Description string + Stakeholders []Stakeholder `gorm:"many2many:StakeholderGroupStakeholder;constraint:OnDelete:CASCADE"` + MigrationWaves []MigrationWave `gorm:"many2many:MigrationWaveStakeholderGroups;constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"many2many:AssessmentStakeholderGroups;constraint:OnDelete:CASCADE"` + Archetypes []Archetype `gorm:"many2many:ArchetypeStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type MigrationWave struct { + Model + Name string `gorm:"uniqueIndex:MigrationWaveA"` + StartDate time.Time `gorm:"uniqueIndex:MigrationWaveA"` + EndDate time.Time `gorm:"uniqueIndex:MigrationWaveA"` + Applications []Application `gorm:"constraint:OnDelete:SET NULL"` + Stakeholders []Stakeholder `gorm:"many2many:MigrationWaveStakeholders;constraint:OnDelete:CASCADE"` + StakeholderGroups []StakeholderGroup `gorm:"many2many:MigrationWaveStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type Archetype struct { + Model + Name string + Description string + Comments string + Review *Review `gorm:"constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"constraint:OnDelete:CASCADE"` + CriteriaTags []Tag `gorm:"many2many:ArchetypeCriteriaTags;constraint:OnDelete:CASCADE"` + Tags []Tag `gorm:"many2many:ArchetypeTags;constraint:OnDelete:CASCADE"` + Stakeholders []Stakeholder `gorm:"many2many:ArchetypeStakeholders;constraint:OnDelete:CASCADE"` + StakeholderGroups []StakeholderGroup `gorm:"many2many:ArchetypeStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type Tag struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"uniqueIndex:tagA;not null"` + Username string + CategoryID uint `gorm:"uniqueIndex:tagA;index;not null"` + Category TagCategory +} + +type TagCategory struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"index;unique;not null"` + Username string + Rank uint + Color string + Tags []Tag `gorm:"foreignKey:CategoryID;constraint:OnDelete:CASCADE"` +} + +type Ticket struct { + Model + // Kind of ticket in the external tracker. + Kind string `gorm:"not null"` + // Parent resource that this ticket should belong to in the tracker. (e.g. Jira project) + Parent string `gorm:"not null"` + // Custom fields to send to the tracker when creating the ticket + Fields JSON `gorm:"type:json"` + // Whether the last attempt to do something with the ticket reported an error + Error bool + // Error message, if any + Message string + // Whether the ticket was created in the external tracker + Created bool + // Reference id in external tracker + Reference string + // URL to ticket in external tracker + Link string + // Status of ticket in external tracker + Status string + LastUpdated time.Time + Application *Application + ApplicationID uint `gorm:"uniqueIndex:ticketA;not null"` + Tracker *Tracker + TrackerID uint `gorm:"uniqueIndex:ticketA;not null"` +} + +type Tracker struct { + Model + Name string `gorm:"index;unique;not null"` + URL string + Kind string + Identity *Identity + IdentityID uint + Connected bool + LastUpdated time.Time + Message string + Insecure bool + Tickets []Ticket +} + +type Import struct { + Model + Filename string + ApplicationName string + BusinessService string + Comments string + Dependency string + DependencyDirection string + Description string + ErrorMessage string + IsValid bool + RecordType1 string + ImportSummary ImportSummary + ImportSummaryID uint `gorm:"index"` + Processed bool + ImportTags []ImportTag `gorm:"constraint:OnDelete:CASCADE"` + BinaryGroup string + BinaryArtifact string + BinaryVersion string + BinaryPackaging string + RepositoryKind string + RepositoryURL string + RepositoryBranch string + RepositoryPath string + Owner string + Contributors string +} + +func (r *Import) AsMap() (m map[string]interface{}) { + m = make(map[string]interface{}) + m["filename"] = r.Filename + m["applicationName"] = r.ApplicationName + // "Application Name" is necessary in order for + // the UI to display the error report correctly. + m["Application Name"] = r.ApplicationName + m["businessService"] = r.BusinessService + m["comments"] = r.Comments + m["dependency"] = r.Dependency + m["dependencyDirection"] = r.DependencyDirection + m["description"] = r.Description + m["errorMessage"] = r.ErrorMessage + m["isValid"] = r.IsValid + m["processed"] = r.Processed + m["recordType1"] = r.RecordType1 + for i, tag := range r.ImportTags { + m[fmt.Sprintf("category%v", i+1)] = tag.Category + m[fmt.Sprintf("tag%v", i+1)] = tag.Name + } + return +} + +type ImportSummary struct { + Model + Content []byte + Filename string + ImportStatus string + Imports []Import `gorm:"constraint:OnDelete:CASCADE"` + CreateEntities bool +} + +type ImportTag struct { + Model + Name string + Category string + ImportID uint `gorm:"index"` + Import *Import +} diff --git a/migration/v12/model/assessment.go b/migration/v12/model/assessment.go new file mode 100644 index 00000000..a8cc0b6f --- /dev/null +++ b/migration/v12/model/assessment.go @@ -0,0 +1,47 @@ +package model + +type Questionnaire struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"unique"` + Description string + Required bool + Sections JSON `gorm:"type:json"` + Thresholds JSON `gorm:"type:json"` + RiskMessages JSON `gorm:"type:json"` + Assessments []Assessment `gorm:"constraint:OnDelete:CASCADE"` +} + +// +// Builtin returns true if this is a Konveyor-provided questionnaire. +func (r *Questionnaire) Builtin() bool { + return r.UUID != nil +} + +type Assessment struct { + Model + ApplicationID *uint `gorm:"uniqueIndex:AssessmentA"` + Application *Application + ArchetypeID *uint `gorm:"uniqueIndex:AssessmentB"` + Archetype *Archetype + QuestionnaireID uint `gorm:"uniqueIndex:AssessmentA;uniqueIndex:AssessmentB"` + Questionnaire Questionnaire + Sections JSON `gorm:"type:json"` + Thresholds JSON `gorm:"type:json"` + RiskMessages JSON `gorm:"type:json"` + Stakeholders []Stakeholder `gorm:"many2many:AssessmentStakeholders;constraint:OnDelete:CASCADE"` + StakeholderGroups []StakeholderGroup `gorm:"many2many:AssessmentStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type Review struct { + Model + BusinessCriticality uint `gorm:"not null"` + EffortEstimate string `gorm:"not null"` + ProposedAction string `gorm:"not null"` + WorkPriority uint `gorm:"not null"` + Comments string + ApplicationID *uint `gorm:"uniqueIndex"` + Application *Application + ArchetypeID *uint `gorm:"uniqueIndex"` + Archetype *Archetype +} diff --git a/migration/v12/model/core.go b/migration/v12/model/core.go new file mode 100644 index 00000000..6fb8ce10 --- /dev/null +++ b/migration/v12/model/core.go @@ -0,0 +1,379 @@ +package model + +import ( + "encoding/json" + "fmt" + "github.com/google/uuid" + liberr "github.com/jortel/go-utils/error" + "github.com/konveyor/tackle2-hub/encryption" + "gorm.io/gorm" + "os" + "path" + "time" +) + +// +// Model Base model. +type Model struct { + ID uint `gorm:"<-:create;primaryKey"` + CreateTime time.Time `gorm:"<-:create;autoCreateTime"` + CreateUser string `gorm:"<-:create"` + UpdateUser string +} + +type Setting struct { + Model + Key string `gorm:"<-:create;uniqueIndex"` + Value JSON `gorm:"type:json"` +} + +// +// With updates the value of the Setting with the json representation +// of the `value` parameter. +func (r *Setting) With(value interface{}) (err error) { + r.Value, err = json.Marshal(value) + if err != nil { + err = liberr.Wrap(err) + } + return +} + +// +// As unmarshalls the value of the Setting into the `ptr` parameter. +func (r *Setting) As(ptr interface{}) (err error) { + err = json.Unmarshal(r.Value, ptr) + if err != nil { + err = liberr.Wrap(err) + } + return +} + +type Bucket struct { + Model + Path string `gorm:"<-:create;uniqueIndex"` + Expiration *time.Time +} + +func (m *Bucket) BeforeCreate(db *gorm.DB) (err error) { + if m.Path == "" { + uid := uuid.New() + m.Path = path.Join( + Settings.Hub.Bucket.Path, + uid.String()) + err = os.MkdirAll(m.Path, 0777) + if err != nil { + err = liberr.Wrap( + err, + "path", + m.Path) + } + } + return +} + +type BucketOwner struct { + BucketID *uint `gorm:"index" ref:"bucket"` + Bucket *Bucket +} + +func (m *BucketOwner) BeforeCreate(db *gorm.DB) (err error) { + if !m.HasBucket() { + b := &Bucket{} + err = db.Create(b).Error + m.SetBucket(&b.ID) + } + return +} + +func (m *BucketOwner) SetBucket(id *uint) { + m.BucketID = id + m.Bucket = nil +} + +func (m *BucketOwner) HasBucket() (b bool) { + return m.BucketID != nil +} + +type File struct { + Model + Name string + Path string `gorm:"<-:create;uniqueIndex"` + Expiration *time.Time +} + +func (m *File) BeforeCreate(db *gorm.DB) (err error) { + uid := uuid.New() + m.Path = path.Join( + Settings.Hub.Bucket.Path, + ".file", + uid.String()) + err = os.MkdirAll(path.Dir(m.Path), 0777) + if err != nil { + err = liberr.Wrap( + err, + "path", + m.Path) + } + return +} + +type Task struct { + Model + BucketOwner + Name string `gorm:"index"` + Addon string `gorm:"index"` + Locator string `gorm:"index"` + Priority int + Image string + Variant string + Policy string + TTL JSON + Data JSON + Started *time.Time + Terminated *time.Time + State string `gorm:"index"` + Errors JSON + Pod string `gorm:"index"` + Retries int + Canceled bool + Report *TaskReport `gorm:"constraint:OnDelete:CASCADE"` + ApplicationID *uint + Application *Application + TaskGroupID *uint `gorm:"<-:create"` + TaskGroup *TaskGroup +} + +func (m *Task) Reset() { + m.Started = nil + m.Terminated = nil + m.Report = nil + m.Errors = nil +} + +func (m *Task) BeforeCreate(db *gorm.DB) (err error) { + err = m.BucketOwner.BeforeCreate(db) + m.Reset() + return +} + +// +// Error appends an error. +func (m *Task) Error(severity, description string, x ...interface{}) { + var list []TaskError + description = fmt.Sprintf(description, x...) + te := TaskError{Severity: severity, Description: description} + _ = json.Unmarshal(m.Errors, &list) + list = append(list, te) + m.Errors, _ = json.Marshal(list) +} + +// +// Map alias. +type Map = map[string]interface{} + +// +// TTL time-to-live. +type TTL struct { + Created int `json:"created,omitempty"` + Pending int `json:"pending,omitempty"` + Postponed int `json:"postponed,omitempty"` + Running int `json:"running,omitempty"` + Succeeded int `json:"succeeded,omitempty"` + Failed int `json:"failed,omitempty"` +} + +// +// TaskError used in Task.Errors. +type TaskError struct { + Severity string `json:"severity"` + Description string `json:"description"` +} + +type TaskReport struct { + Model + Status string + Errors JSON + Total int + Completed int + Activity JSON `gorm:"type:json"` + Attached JSON `gorm:"type:json" ref:"[]file"` + Result JSON `gorm:"type:json"` + TaskID uint `gorm:"<-:create;uniqueIndex"` + Task *Task +} + +type TaskGroup struct { + Model + BucketOwner + Name string + Addon string + Data JSON + Tasks []Task `gorm:"constraint:OnDelete:CASCADE"` + List JSON + State string +} + +// +// Propagate group data into the task. +func (m *TaskGroup) Propagate() (err error) { + for i := range m.Tasks { + task := &m.Tasks[i] + task.State = m.State + task.SetBucket(m.BucketID) + if task.Addon == "" { + task.Addon = m.Addon + } + if m.Data == nil { + continue + } + a := Map{} + err = json.Unmarshal(m.Data, &a) + if err != nil { + err = liberr.Wrap( + err, + "id", + m.ID) + return + } + b := Map{} + err = json.Unmarshal(task.Data, &b) + if err != nil { + err = liberr.Wrap( + err, + "id", + m.ID) + return + } + task.Data, _ = json.Marshal(m.merge(a, b)) + } + + return +} + +// +// merge maps B into A. +// The B map is the authority. +func (m *TaskGroup) merge(a, b Map) (out Map) { + if a == nil { + a = Map{} + } + if b == nil { + b = Map{} + } + out = Map{} + // + // Merge-in elements found in B and in A. + for k, v := range a { + out[k] = v + if bv, found := b[k]; found { + out[k] = bv + if av, cast := v.(Map); cast { + if bv, cast := bv.(Map); cast { + out[k] = m.merge(av, bv) + } else { + out[k] = bv + } + } + } + } + // + // Add elements found only in B. + for k, v := range b { + if _, found := a[k]; !found { + out[k] = v + } + } + + return +} + +// +// Proxy configuration. +// kind = (http|https) +type Proxy struct { + Model + Enabled bool + Kind string `gorm:"uniqueIndex"` + Host string `gorm:"not null"` + Port int + Excluded JSON `gorm:"type:json"` + IdentityID *uint `gorm:"index"` + Identity *Identity +} + +// Identity represents and identity with a set of credentials. +type Identity struct { + Model + Kind string `gorm:"not null"` + Name string `gorm:"index;unique;not null"` + Description string + User string + Password string + Key string + Settings string + Proxies []Proxy `gorm:"constraint:OnDelete:SET NULL"` +} + +// Encrypt sensitive fields. +// The ref identity is used to determine when sensitive fields +// have changed and need to be (re)encrypted. +func (r *Identity) Encrypt(ref *Identity) (err error) { + passphrase := Settings.Encryption.Passphrase + aes := encryption.New(passphrase) + if r.Password != ref.Password { + if r.Password != "" { + r.Password, err = aes.Encrypt(r.Password) + if err != nil { + err = liberr.Wrap(err) + return + } + } + } + if r.Key != ref.Key { + if r.Key != "" { + r.Key, err = aes.Encrypt(r.Key) + if err != nil { + err = liberr.Wrap(err) + return + } + } + } + if r.Settings != ref.Settings { + if r.Settings != "" { + r.Settings, err = aes.Encrypt(r.Settings) + if err != nil { + err = liberr.Wrap(err) + return + } + } + } + return +} + +// Decrypt sensitive fields. +func (r *Identity) Decrypt() (err error) { + passphrase := Settings.Encryption.Passphrase + aes := encryption.New(passphrase) + if r.Password != "" { + r.Password, err = aes.Decrypt(r.Password) + if err != nil { + err = liberr.Wrap(err) + return + } + } + if r.Key != "" { + r.Key, err = aes.Decrypt(r.Key) + if err != nil { + err = liberr.Wrap(err) + return + } + } + if r.Settings != "" { + r.Settings, err = aes.Decrypt(r.Settings) + if err != nil { + err = liberr.Wrap(err) + return + } + } + return +} diff --git a/migration/v12/model/pkg.go b/migration/v12/model/pkg.go new file mode 100644 index 00000000..46c071c5 --- /dev/null +++ b/migration/v12/model/pkg.go @@ -0,0 +1,55 @@ +package model + +import "github.com/konveyor/tackle2-hub/settings" + +var ( + Settings = &settings.Settings +) + +// +// JSON field (data) type. +type JSON = []byte + +// +// All builds all models. +// Models are enumerated such that each are listed after +// all the other models on which they may depend. +func All() []interface{} { + return []interface{}{ + Application{}, + TechDependency{}, + Incident{}, + Analysis{}, + Issue{}, + Bucket{}, + BusinessService{}, + Dependency{}, + File{}, + Fact{}, + Identity{}, + Import{}, + ImportSummary{}, + ImportTag{}, + JobFunction{}, + MigrationWave{}, + Proxy{}, + Review{}, + Setting{}, + RuleSet{}, + Rule{}, + Stakeholder{}, + StakeholderGroup{}, + Tag{}, + TagCategory{}, + Target{}, + Task{}, + TaskGroup{}, + TaskReport{}, + Ticket{}, + Tracker{}, + ApplicationTag{}, + Questionnaire{}, + Assessment{}, + Archetype{}, + } +} diff --git a/model/pkg.go b/model/pkg.go index 9f9d6fc0..17524f96 100644 --- a/model/pkg.go +++ b/model/pkg.go @@ -1,7 +1,7 @@ package model import ( - "github.com/konveyor/tackle2-hub/migration/v11/model" + "github.com/konveyor/tackle2-hub/migration/v12/model" "gorm.io/datatypes" ) diff --git a/reaper/file.go b/reaper/file.go index 350be2e9..2f0a6874 100644 --- a/reaper/file.go +++ b/reaper/file.go @@ -66,6 +66,7 @@ func (r *FileReaper) busy(file *model.File) (busy bool, err error) { var n int64 ref := RefCounter{DB: r.DB} for _, m := range []interface{}{ + &model.TaskReport{}, &model.RuleSet{}, &model.Rule{}, &model.Target{}, diff --git a/reaper/ref.go b/reaper/ref.go index 90ca0f5b..1c0a9f42 100644 --- a/reaper/ref.go +++ b/reaper/ref.go @@ -4,6 +4,7 @@ import ( liberr "github.com/jortel/go-utils/error" "gorm.io/gorm" "reflect" + "fmt" ) // @@ -19,11 +20,29 @@ type RefCounter struct { func (r *RefCounter) Count(m interface{}, kind string, pk uint) (nRef int64, err error) { db := r.DB.Model(m) fields := 0 + j := 0 add := func(ft reflect.StructField) { tag, found := ft.Tag.Lookup("ref") if found && tag == kind { db = db.Or(ft.Name, pk) fields++ + return + } + if found && tag == "[]"+kind { + db = db.Joins( + fmt.Sprintf( + ",json_each(%s) j%d", + ft.Name, + j)) + db = db.Or( + fmt.Sprintf( + "json_extract(j%d.value,?)=?", + j), + "$.id", + pk) + fields++ + j++ + return } } var find func(interface{}) @@ -56,6 +75,8 @@ func (r *RefCounter) Count(m interface{}, kind string, pk uint) (nRef int64, err } case reflect.Uint: add(ft) + case reflect.Slice: + add(ft) } } } diff --git a/test/api/file/api_test.go b/test/api/file/api_test.go index 68ca9801..84a4d0ad 100644 --- a/test/api/file/api_test.go +++ b/test/api/file/api_test.go @@ -7,6 +7,9 @@ import ( "testing" "github.com/konveyor/tackle2-hub/test/assert" + "k8s.io/apimachinery/pkg/util/rand" + "io/ioutil" + "strings" ) func TestFilePutGetDelete(t *testing.T) { @@ -43,3 +46,66 @@ func TestFilePutGetDelete(t *testing.T) { }) } } + +func TestFileTouchPatchGetDelete(t *testing.T) { + for _, r := range Samples { + t.Run(r.Name, func(t *testing.T) { + // Touch. + name := "Patch-Test" + file, err := File.Touch(name) + if err != nil { + t.Errorf(err.Error()) + } + // Patch (append) + content := "This is my Test. " + for _, p := range strings.Fields(content) { + err = File.Patch(file.ID, []byte(p+" ")) + if err != nil { + t.Errorf(err.Error()) + } + } + // Get. + tmp := fmt.Sprintf( + "/tmp/%s-%d", + r.Name, + rand.Int()) + err = File.Get(file.ID, tmp) + if err != nil { + t.Errorf(err.Error()) + } + defer func() { + _ = os.Remove(tmp) + }() + if file.Name != name { + t.Errorf( + "File name mismatch. Expected: '%s' found: '%s'", + name, + file.Name) + } + + f, err := os.Open(tmp) + if err != nil { + t.Errorf(err.Error()) + } + read, err := ioutil.ReadAll(f) + if err != nil { + t.Errorf(err.Error()) + } + if content != string(read) { + t.Errorf( + "File content mismatch. Expcected: '%s' read: '%s'", + content, + string(read)) + } + // Delete. + err = File.Delete(file.ID) + if err != nil { + t.Errorf(err.Error()) + } + err = File.Get(file.ID, "/dev/null") + if err == nil { + t.Errorf("Resource exits, but should be deleted: %v", r) + } + }) + } +}