Skip to content

Commit

Permalink
✨ Task report files (#577)
Browse files Browse the repository at this point in the history 
Add support for:
- TaskReport attached files. The addon (and tackle2-addon command) can
attach files with command output and/or log files.
- File API to support creating empty files.
- File API to support append content - PATCH /files/:id.
- Addon (adapter) to support attach files to the task report.

Reaper updated to support fields with array of refs. New tag
`ref=[]file`

Example:
```
 - '[CMD] Running: /usr/bin/konveyor-analyzer --provider-settings /addon/opt/settings.json --output-file /addon/report.yaml --no-dependency-rules --rules /addon/rules/rulesets/1/rules --rules /addon/rules/rulesets/24/rules --rules /addon/rules/rulesets/20/rules --label-selector konveyor.io/target=cloud-readiness --dep-label-selector !konveyor.io/dep-source=open-source'
    - '[CMD] /usr/bin/konveyor-analyzer succeeded.'
    - '[CMD] Running: /usr/bin/konveyor-analyzer-dep --provider-settings /addon/opt/settings.json --output-file /addon/deps.yaml'
    - '[CMD] /usr/bin/konveyor-analyzer-dep succeeded.'
    - 'Analysis reported. duration: 102.5784ms'
    - '[TAG] Tagging Application 4.'
    - Facts updated.
    - Done.
attached:
    - id: 989
      name: konveyor-analyzer.output
      activity: 44
    - id: 991
      name: konveyor-analyzer-dep.output
      activity: 50
```

---------

Signed-off-by: Jeff Ortel <jortel@redhat.com>
  • Loading branch information
@jortel
jortel committed Dec 19, 2023
1 parent d8e71ec commit f09d0b2
Show file tree
Hide file tree
Showing 16 changed files with 1,324 additions and 34 deletions.
38 changes: 38 additions & 0 deletions addon/task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,44 @@ func (h *Task) Activity(entry string, v ...interface{}) {
return
}

//
// Attach ensures the file is attached to the report
// associated with the last entry in the activity.
func (h *Task) Attach(f *api.File) {
index := len(h.report.Activity)
h.AttachAt(f, index)
return
}

//
// AttachAt ensures the file is attached to
// the report indexed to the activity.
// The activity is a 1-based index. Zero(0) means NOT associated.
func (h *Task) AttachAt(f *api.File, activity int) {
for _, ref := range h.report.Attached {
if ref.ID == f.ID {
return
}
}
Log.Info(
"Addon attached: %s",
"path",
f.Path,
"activity",
activity)
h.report.Attached = append(
h.report.Attached,
api.Attachment{
Activity: activity,
Ref: api.Ref{
ID: f.ID,
Name: f.Name,
},
})
h.pushReport()
return
}

//
// Total report addon total items.
func (h *Task) Total(n int) {
Expand Down
60 changes: 60 additions & 0 deletions api/file.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func (h FileHandler) AddRoutes(e *gin.Engine) {
routeGroup.GET(FilesRoot+"/", h.List)
routeGroup.POST(FileRoot, h.Create)
routeGroup.PUT(FileRoot, h.Create)
routeGroup.PATCH(FileRoot, h.Append)
routeGroup.GET(FileRoot, h.Get)
routeGroup.DELETE(FileRoot, h.Delete)
}
Expand Down Expand Up @@ -122,6 +123,65 @@ func (h FileHandler) Create(ctx *gin.Context) {
h.Respond(ctx, http.StatusCreated, r)
}

// Append godoc
// @summary Append a file.
// @description Append a file.
// @tags file
// @accept json
// @produce json
// @success 204
// @router /files/{id} [put]
// @param name id uint true "File ID"
func (h FileHandler) Append(ctx *gin.Context) {
var err error
input, err := ctx.FormFile(FileField)
if err != nil {
err = &BadRequestError{err.Error()}
_ = ctx.Error(err)
return
}
m := &model.File{}
id := h.pk(ctx)
db := h.DB(ctx)
err = db.First(m, id).Error
if err != nil {
_ = ctx.Error(err)
return
}
reader, err := input.Open()
if err != nil {
err = &BadRequestError{err.Error()}
_ = ctx.Error(err)
return
}
defer func() {
_ = reader.Close()
}()
writer, err := os.OpenFile(m.Path, os.O_WRONLY|os.O_APPEND, 0666)
if err != nil {
_ = ctx.Error(err)
return
}
defer func() {
_ = writer.Close()
}()
_, err = io.Copy(writer, reader)
if err != nil {
_ = ctx.Error(err)
return
}
db = h.DB(ctx)
db = db.Model(m)
user := h.BaseHandler.CurrentUser(ctx)
err = db.Update("UpdateUser", user).Error
if err != nil {
_ = ctx.Error(err)
return
}

h.Status(ctx, http.StatusNoContent)
}

// Get godoc
// @summary Get a file by ID.
// @description Get a file by ID. Returns api.File when Accept=application/json else the file content.
Expand Down
126 changes: 99 additions & 27 deletions api/task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,13 @@ import (
tasking "github.com/konveyor/tackle2-hub/task"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"io/ioutil"
k8serr "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/utils/strings/slices"
"net/http"
"sort"
"strconv"
"strings"
"time"
)

Expand Down Expand Up @@ -83,6 +88,14 @@ func (h TaskHandler) Get(ctx *gin.Context) {
}
r := Task{}
r.With(task)
q := ctx.Query("merged")
if b, _ := strconv.ParseBool(q); b {
err := r.injectFiles(h.DB(ctx))
if err != nil {
_ = ctx.Error(result.Error)
return
}
}

h.Respond(ctx, http.StatusOK, r)
}
Expand Down Expand Up @@ -510,26 +523,27 @@ type TaskError struct {
// Task REST resource.
type Task struct {
Resource `yaml:",inline"`
Name string `json:"name"`
Locator string `json:"locator,omitempty" yaml:",omitempty"`
Priority int `json:"priority,omitempty" yaml:",omitempty"`
Variant string `json:"variant,omitempty" yaml:",omitempty"`
Policy string `json:"policy,omitempty" yaml:",omitempty"`
TTL *TTL `json:"ttl,omitempty" yaml:",omitempty"`
Addon string `json:"addon,omitempty" binding:"required" yaml:",omitempty"`
Data interface{} `json:"data" swaggertype:"object" binding:"required"`
Application *Ref `json:"application,omitempty" yaml:",omitempty"`
State string `json:"state"`
Image string `json:"image,omitempty" yaml:",omitempty"`
Pod string `json:"pod,omitempty" yaml:",omitempty"`
Retries int `json:"retries,omitempty" yaml:",omitempty"`
Started *time.Time `json:"started,omitempty" yaml:",omitempty"`
Terminated *time.Time `json:"terminated,omitempty" yaml:",omitempty"`
Canceled bool `json:"canceled,omitempty" yaml:",omitempty"`
Bucket *Ref `json:"bucket,omitempty" yaml:",omitempty"`
Purged bool `json:"purged,omitempty" yaml:",omitempty"`
Errors []TaskError `json:"errors,omitempty" yaml:",omitempty"`
Activity []string `json:"activity,omitempty" yaml:",omitempty"`
Name string `json:"name"`
Locator string `json:"locator,omitempty" yaml:",omitempty"`
Priority int `json:"priority,omitempty" yaml:",omitempty"`
Variant string `json:"variant,omitempty" yaml:",omitempty"`
Policy string `json:"policy,omitempty" yaml:",omitempty"`
TTL *TTL `json:"ttl,omitempty" yaml:",omitempty"`
Addon string `json:"addon,omitempty" binding:"required" yaml:",omitempty"`
Data interface{} `json:"data" swaggertype:"object" binding:"required"`
Application *Ref `json:"application,omitempty" yaml:",omitempty"`
State string `json:"state"`
Image string `json:"image,omitempty" yaml:",omitempty"`
Pod string `json:"pod,omitempty" yaml:",omitempty"`
Retries int `json:"retries,omitempty" yaml:",omitempty"`
Started *time.Time `json:"started,omitempty" yaml:",omitempty"`
Terminated *time.Time `json:"terminated,omitempty" yaml:",omitempty"`
Canceled bool `json:"canceled,omitempty" yaml:",omitempty"`
Bucket *Ref `json:"bucket,omitempty" yaml:",omitempty"`
Purged bool `json:"purged,omitempty" yaml:",omitempty"`
Errors []TaskError `json:"errors,omitempty" yaml:",omitempty"`
Activity []string `json:"activity,omitempty" yaml:",omitempty"`
Attached []Attachment `json:"attached" yaml:",omitempty"`
}

//
Expand Down Expand Up @@ -563,6 +577,7 @@ func (r *Task) With(m *model.Task) {
report.With(m.Report)
r.Activity = report.Activity
r.Errors = append(report.Errors, r.Errors...)
r.Attached = report.Attached
switch r.State {
case tasking.Succeeded:
switch report.Status {
Expand Down Expand Up @@ -594,17 +609,58 @@ func (r *Task) Model() (m *model.Task) {
return
}

//
// injectFiles inject attached files into the activity.
func (r *Task) injectFiles(db *gorm.DB) (err error) {
sort.Slice(
r.Attached,
func(i, j int) bool {
return r.Attached[i].Activity > r.Attached[j].Activity
})
for _, ref := range r.Attached {
if ref.Activity == 0 {
continue
}
if ref.Activity > len(r.Activity) {
continue
}
m := &model.File{}
err = db.First(m, ref.ID).Error
if err != nil {
return
}
b, nErr := ioutil.ReadFile(m.Path)
if nErr != nil {
err = nErr
return
}
var content []string
for _, s := range strings.Split(string(b), "\n") {
content = append(
content,
"> "+s)
}
snipA := slices.Clone(r.Activity[:ref.Activity])
snipB := slices.Clone(r.Activity[ref.Activity:])
r.Activity = append(
append(snipA, content...),
snipB...)
}
return
}

//
// TaskReport REST resource.
type TaskReport struct {
Resource `yaml:",inline"`
Status string `json:"status"`
Errors []TaskError `json:"errors,omitempty" yaml:",omitempty"`
Total int `json:"total,omitempty" yaml:",omitempty"`
Completed int `json:"completed,omitempty" yaml:",omitempty"`
Activity []string `json:"activity,omitempty" yaml:",omitempty"`
Result interface{} `json:"result,omitempty" yaml:",omitempty" swaggertype:"object"`
TaskID uint `json:"task"`
Status string `json:"status"`
Errors []TaskError `json:"errors,omitempty" yaml:",omitempty"`
Total int `json:"total,omitempty" yaml:",omitempty"`
Completed int `json:"completed,omitempty" yaml:",omitempty"`
Activity []string `json:"activity,omitempty" yaml:",omitempty"`
Attached []Attachment `json:"attached,omitempty" yaml:",omitempty"`
Result interface{} `json:"result,omitempty" yaml:",omitempty" swaggertype:"object"`
TaskID uint `json:"task"`
}

//
Expand All @@ -621,6 +677,9 @@ func (r *TaskReport) With(m *model.TaskReport) {
if m.Errors != nil {
_ = json.Unmarshal(m.Errors, &r.Errors)
}
if m.Attached != nil {
_ = json.Unmarshal(m.Attached, &r.Attached)
}
if m.Result != nil {
_ = json.Unmarshal(m.Result, &r.Result)
}
Expand All @@ -647,7 +706,20 @@ func (r *TaskReport) Model() (m *model.TaskReport) {
if r.Errors != nil {
m.Errors, _ = json.Marshal(r.Errors)
}
if r.Attached != nil {
m.Attached, _ = json.Marshal(r.Attached)
}
m.ID = r.ID

return
}

//
// Attachment associates Files with a TaskReport.
type Attachment struct {
// Ref references an attached File.
Ref `yaml:",inline"`
// Activity index (1-based) association with an
// activity entry. Zero(0) indicates not associated.
Activity int `json:"activity,omitempty" yaml:",omitempty"`
}
46 changes: 40 additions & 6 deletions binding/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -422,9 +422,19 @@ func (r *Client) FileGet(path, destination string) (err error) {
}

//
// FilePut uploads a file.
// FilePost uploads a file.
// Returns the created File resource.
func (r *Client) FilePut(path, source string, object interface{}) (err error) {
func (r *Client) FilePost(path, source string, object interface{}) (err error) {
if source == "" {
fields := []Field{
{
Name: api.FileField,
Reader: bytes.NewReader([]byte{}),
},
}
err = r.FileSend(path, http.MethodPost, fields, object)
return
}
isDir, nErr := r.IsDir(source, true)
if nErr != nil {
err = nErr
Expand All @@ -440,14 +450,24 @@ func (r *Client) FilePut(path, source string, object interface{}) (err error) {
Path: source,
},
}
err = r.FileSend(path, http.MethodPut, fields, object)
err = r.FileSend(path, http.MethodPost, fields, object)
return
}

//
// FilePost uploads a file.
// FilePut uploads a file.
// Returns the created File resource.
func (r *Client) FilePost(path, source string, object interface{}) (err error) {
func (r *Client) FilePut(path, source string, object interface{}) (err error) {
if source == "" {
fields := []Field{
{
Name: api.FileField,
Reader: bytes.NewReader([]byte{}),
},
}
err = r.FileSend(path, http.MethodPut, fields, object)
return
}
isDir, nErr := r.IsDir(source, true)
if nErr != nil {
err = nErr
Expand All @@ -463,7 +483,21 @@ func (r *Client) FilePost(path, source string, object interface{}) (err error) {
Path: source,
},
}
err = r.FileSend(path, http.MethodPost, fields, object)
err = r.FileSend(path, http.MethodPut, fields, object)
return
}

//
// FilePatch appends file.
// Returns the created File resource.
func (r *Client) FilePatch(path string, buffer []byte) (err error) {
fields := []Field{
{
Name: api.FileField,
Reader: bytes.NewReader(buffer),
},
}
err = r.FileSend(path, http.MethodPatch, fields, nil)
return
}

Expand Down
Loading

0 comments on commit f09d0b2

Please sign in to comment.