Skip to content

Commit

Permalink
database: add namespace to item (#862)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhenghaoz
zhenghaoz authored Oct 11, 2024
1 parent 5a3d1d1 commit 6d3444e
Show file tree
Hide file tree
Showing 11 changed files with 394 additions and 242 deletions.
8 changes: 4 additions & 4 deletions master/rest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -807,7 +807,7 @@ func (m *Master) getRecommend(request *restful.Request, response *restful.Respon
// Send result
details := make([]data.Item, len(results))
for i := range results {
details[i], err = m.DataClient.GetItem(ctx, results[i])
details[i], err = m.DataClient.GetItem(ctx, "", results[i])
if err != nil {
server.InternalServerError(response, err)
return
Expand All @@ -832,7 +832,7 @@ func (m *Master) getTypedFeedbackByUser(request *restful.Request, response *rest
}
feedbackType := request.PathParameter("feedback-type")
userId := request.PathParameter("user-id")
feedback, err := m.DataClient.GetUserFeedback(ctx, userId, m.Config.Now(), feedbackType)
feedback, err := m.DataClient.GetUserFeedback(ctx, "", userId, m.Config.Now(), feedbackType)
if err != nil {
server.InternalServerError(response, err)
return
Expand All @@ -843,7 +843,7 @@ func (m *Master) getTypedFeedbackByUser(request *restful.Request, response *rest
details[i].UserId = feedback[i].UserId
details[i].Timestamp = feedback[i].Timestamp
details[i].Comment = feedback[i].Comment
details[i].Item, err = m.DataClient.GetItem(ctx, feedback[i].ItemId)
details[i].Item, err = m.DataClient.GetItem(ctx, "", feedback[i].ItemId)
if errors.Is(err, errors.NotFound) {
details[i].Item = data.Item{ItemId: feedback[i].ItemId, Comment: "** This item doesn't exist in Gorse **"}
} else if err != nil {
Expand Down Expand Up @@ -895,7 +895,7 @@ func (m *Master) searchDocuments(collection, subset, category string, request *r
details := make([]ScoredItem, len(scores))
for i := range scores {
details[i].Score = scores[i].Score
details[i].Item, err = m.DataClient.GetItem(ctx, scores[i].Id)
details[i].Item, err = m.DataClient.GetItem(ctx, "", scores[i].Id)
if err != nil {
server.InternalServerError(response, err)
return
Expand Down
2 changes: 1 addition & 1 deletion master/tasks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1368,7 +1368,7 @@ func (t *CacheGarbageCollectionTask) run(ctx context.Context, j *task.JobsAlloca
return nil
}
// check item in database
_, err := t.DataClient.GetItem(ctx, itemId)
_, err := t.DataClient.GetItem(ctx, "", itemId)
if !errors.Is(err, errors.NotFound) {
if err != nil {
log.Logger().Error("failed to load item", zap.String("item_id", itemId), zap.Error(err))
Expand Down
36 changes: 18 additions & 18 deletions server/rest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -596,7 +596,7 @@ func (s *RestServer) searchDocuments(collection, subset, category string, isItem

readItems := mapset.NewSet[string]()
if userId != "" {
feedback, err := s.DataClient.GetUserFeedback(ctx, userId, s.Config.Now())
feedback, err := s.DataClient.GetUserFeedback(ctx, "", userId, s.Config.Now())
if err != nil {
InternalServerError(response, err)
return
Expand Down Expand Up @@ -656,7 +656,7 @@ func (s *RestServer) getTypedFeedbackByItem(request *restful.Request, response *
}
feedbackType := request.PathParameter("feedback-type")
itemId := request.PathParameter("item-id")
feedback, err := s.DataClient.GetItemFeedback(ctx, itemId, feedbackType)
feedback, err := s.DataClient.GetItemFeedback(ctx, "", itemId, feedbackType)
if err != nil {
InternalServerError(response, err)
return
Expand All @@ -671,7 +671,7 @@ func (s *RestServer) getFeedbackByItem(request *restful.Request, response *restf
ctx = request.Request.Context()
}
itemId := request.PathParameter("item-id")
feedback, err := s.DataClient.GetItemFeedback(ctx, itemId)
feedback, err := s.DataClient.GetItemFeedback(ctx, "", itemId)
if err != nil {
InternalServerError(response, err)
return
Expand Down Expand Up @@ -774,7 +774,7 @@ type recommendContext struct {

func (s *RestServer) createRecommendContext(ctx context.Context, userId string, categories []string, n int) (*recommendContext, error) {
// pull historical feedback
userFeedback, err := s.DataClient.GetUserFeedback(ctx, userId, s.Config.Now())
userFeedback, err := s.DataClient.GetUserFeedback(ctx, "", userId, s.Config.Now())
if err != nil {
return nil, errors.Trace(err)
}
Expand Down Expand Up @@ -847,14 +847,14 @@ func (s *RestServer) RecommendUserBased(ctx *recommendContext) error {
}
for _, user := range similarUsers {
// load historical feedback
feedbacks, err := s.DataClient.GetUserFeedback(ctx.context, user.Id, s.Config.Now(), s.Config.Recommend.DataSource.PositiveFeedbackTypes...)
feedbacks, err := s.DataClient.GetUserFeedback(ctx.context, "", user.Id, s.Config.Now(), s.Config.Recommend.DataSource.PositiveFeedbackTypes...)
if err != nil {
return errors.Trace(err)
}
// add unseen items
for _, feedback := range feedbacks {
if !ctx.excludeSet.Contains(feedback.ItemId) {
item, err := s.DataClient.GetItem(ctx.context, feedback.ItemId)
item, err := s.DataClient.GetItem(ctx.context, "", feedback.ItemId)
if err != nil {
return errors.Trace(err)
}
Expand Down Expand Up @@ -1297,7 +1297,7 @@ func (s *RestServer) getTypedFeedbackByUser(request *restful.Request, response *
}
feedbackType := request.PathParameter("feedback-type")
userId := request.PathParameter("user-id")
feedback, err := s.DataClient.GetUserFeedback(ctx, userId, s.Config.Now(), feedbackType)
feedback, err := s.DataClient.GetUserFeedback(ctx, "", userId, s.Config.Now(), feedbackType)
if err != nil {
InternalServerError(response, err)
return
Expand All @@ -1312,7 +1312,7 @@ func (s *RestServer) getFeedbackByUser(request *restful.Request, response *restf
ctx = request.Request.Context()
}
userId := request.PathParameter("user-id")
feedback, err := s.DataClient.GetUserFeedback(ctx, userId, s.Config.Now())
feedback, err := s.DataClient.GetUserFeedback(ctx, "", userId, s.Config.Now())
if err != nil {
InternalServerError(response, err)
return
Expand Down Expand Up @@ -1345,7 +1345,7 @@ func (s *RestServer) batchInsertItems(ctx context.Context, response *restful.Res
)
// load existed items
start := time.Now()
existedItems, err := s.DataClient.BatchGetItems(ctx, lo.Map(temp, func(t Item, i int) string {
existedItems, err := s.DataClient.BatchGetItems(ctx, "", lo.Map(temp, func(t Item, i int) string {
return t.ItemId
}))
if err != nil {
Expand Down Expand Up @@ -1512,7 +1512,7 @@ func (s *RestServer) modifyItem(request *restful.Request, response *restful.Resp
}
}
// modify item
if err := s.DataClient.ModifyItem(ctx, itemId, patch); err != nil {
if err := s.DataClient.ModifyItem(ctx, "", itemId, patch); err != nil {
InternalServerError(response, err)
return
}
Expand Down Expand Up @@ -1556,7 +1556,7 @@ func (s *RestServer) getItem(request *restful.Request, response *restful.Respons
// Get item id
itemId := request.PathParameter("item-id")
// Get item
item, err := s.DataClient.GetItem(ctx, itemId)
item, err := s.DataClient.GetItem(ctx, "", itemId)
if err != nil {
if errors.Is(err, errors.NotFound) {
PageNotFound(response, err)
Expand All @@ -1575,7 +1575,7 @@ func (s *RestServer) deleteItem(request *restful.Request, response *restful.Resp
}
itemId := request.PathParameter("item-id")
// delete item from database
if err := s.DataClient.DeleteItem(ctx, itemId); err != nil {
if err := s.DataClient.DeleteItem(ctx, "", itemId); err != nil {
InternalServerError(response, err)
return
}
Expand All @@ -1596,7 +1596,7 @@ func (s *RestServer) insertItemCategory(request *restful.Request, response *rest
itemId := request.PathParameter("item-id")
category := request.PathParameter("category")
// fetch item
item, err := s.DataClient.GetItem(ctx, itemId)
item, err := s.DataClient.GetItem(ctx, "", itemId)
if err != nil {
InternalServerError(response, err)
return
Expand Down Expand Up @@ -1626,7 +1626,7 @@ func (s *RestServer) deleteItemCategory(request *restful.Request, response *rest
itemId := request.PathParameter("item-id")
category := request.PathParameter("category")
// fetch item
item, err := s.DataClient.GetItem(ctx, itemId)
item, err := s.DataClient.GetItem(ctx, "", itemId)
if err != nil {
InternalServerError(response, err)
return
Expand Down Expand Up @@ -1777,7 +1777,7 @@ func (s *RestServer) getUserItemFeedback(request *restful.Request, response *res
// Parse parameters
userId := request.PathParameter("user-id")
itemId := request.PathParameter("item-id")
if feedback, err := s.DataClient.GetUserItemFeedback(ctx, userId, itemId); err != nil {
if feedback, err := s.DataClient.GetUserItemFeedback(ctx, "", userId, itemId); err != nil {
InternalServerError(response, err)
} else {
Ok(response, feedback)
Expand All @@ -1792,7 +1792,7 @@ func (s *RestServer) deleteUserItemFeedback(request *restful.Request, response *
// Parse parameters
userId := request.PathParameter("user-id")
itemId := request.PathParameter("item-id")
if deleteCount, err := s.DataClient.DeleteUserItemFeedback(ctx, userId, itemId); err != nil {
if deleteCount, err := s.DataClient.DeleteUserItemFeedback(ctx, "", userId, itemId); err != nil {
InternalServerError(response, err)
} else {
Ok(response, Success{RowAffected: deleteCount})
Expand All @@ -1808,7 +1808,7 @@ func (s *RestServer) getTypedUserItemFeedback(request *restful.Request, response
feedbackType := request.PathParameter("feedback-type")
userId := request.PathParameter("user-id")
itemId := request.PathParameter("item-id")
if feedback, err := s.DataClient.GetUserItemFeedback(ctx, userId, itemId, feedbackType); err != nil {
if feedback, err := s.DataClient.GetUserItemFeedback(ctx, "", userId, itemId, feedbackType); err != nil {
InternalServerError(response, err)
} else if feedbackType == "" {
Text(response, "{}")
Expand All @@ -1826,7 +1826,7 @@ func (s *RestServer) deleteTypedUserItemFeedback(request *restful.Request, respo
feedbackType := request.PathParameter("feedback-type")
userId := request.PathParameter("user-id")
itemId := request.PathParameter("item-id")
if deleteCount, err := s.DataClient.DeleteUserItemFeedback(ctx, userId, itemId, feedbackType); err != nil {
if deleteCount, err := s.DataClient.DeleteUserItemFeedback(ctx, "", userId, itemId, feedbackType); err != nil {
InternalServerError(response, err)
} else {
Ok(response, Success{deleteCount})
Expand Down
8 changes: 4 additions & 4 deletions server/rest_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -751,15 +751,15 @@ func (suite *ServerTestSuite) TestFeedback() {
Header("X-API-Key", apiKey).
Expect(t).
Status(http.StatusOK).
Body(`[{"FeedbackType":"click", "UserId": "2", "ItemId": "4", "Timestamp":"0001-01-01T00:00:00Z","Comment":""}]`).
Body(suite.marshal([]data.Feedback{feedback[2]})).
End()
apitest.New().
Handler(suite.handler).
Get("/api/item/4/feedback/click").
Header("X-API-Key", apiKey).
Expect(t).
Status(http.StatusOK).
Body(`[{"FeedbackType":"click", "UserId": "2", "ItemId": "4", "Timestamp":"0001-01-01T00:00:00Z","Comment":""}]`).
Body(suite.marshal([]data.Feedback{feedback[2]})).
End()
// test overwrite
apitest.New().
Expand All @@ -774,7 +774,7 @@ func (suite *ServerTestSuite) TestFeedback() {
Status(http.StatusOK).
Body(`{"RowAffected": 1}`).
End()
ret, err := suite.DataClient.GetUserFeedback(ctx, "0", suite.Config.Now(), "click")
ret, err := suite.DataClient.GetUserFeedback(ctx, "", "0", suite.Config.Now(), "click")
assert.NoError(t, err)
assert.Equal(t, 1, len(ret))
assert.Equal(t, "override", ret[0].Comment)
Expand All @@ -791,7 +791,7 @@ func (suite *ServerTestSuite) TestFeedback() {
Status(http.StatusOK).
Body(`{"RowAffected": 1}`).
End()
ret, err = suite.DataClient.GetUserFeedback(ctx, "0", suite.Config.Now(), "click")
ret, err = suite.DataClient.GetUserFeedback(ctx, "", "0", suite.Config.Now(), "click")
assert.NoError(t, err)
assert.Equal(t, 1, len(ret))
assert.Equal(t, "override", ret[0].Comment)
Expand Down
37 changes: 29 additions & 8 deletions storage/data/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ func ValidateLabels(o any) error {

// Item stores meta data about item.
type Item struct {
Namespace string `gorm:"primaryKey" mapstructure:"namespace"`
ItemId string `gorm:"primaryKey" mapstructure:"item_id"`
IsHidden bool `mapstructure:"is_hidden"`
Categories []string `gorm:"serializer:json" mapstructure:"categories"`
Expand All @@ -116,6 +117,18 @@ type ItemPatch struct {
Comment *string
}

type ItemUID struct {
Namespace string
ItemId string
}

func (item *Item) ItemUID() ItemUID {
return ItemUID{
Namespace: item.Namespace,
ItemId: item.ItemId,
}
}

// User stores meta data about user.
type User struct {
UserId string `gorm:"primaryKey" mapstructure:"user_id"`
Expand All @@ -133,11 +146,19 @@ type UserPatch struct {

// FeedbackKey identifies feedback.
type FeedbackKey struct {
Namespace string `gorm:"column:namespace" mapstructure:"namespace"`
FeedbackType string `gorm:"column:feedback_type" mapstructure:"feedback_type"`
UserId string `gorm:"column:user_id" mapstructure:"user_id"`
ItemId string `gorm:"column:item_id" mapstructure:"item_id"`
}

func (key *FeedbackKey) ItemUID() ItemUID {
return ItemUID{
Namespace: key.Namespace,
ItemId: key.ItemId,
}
}

// Feedback stores feedback.
type Feedback struct {
FeedbackKey `gorm:"embedded" mapstructure:",squash"`
Expand Down Expand Up @@ -225,20 +246,20 @@ type Database interface {
Close() error
Purge() error
BatchInsertItems(ctx context.Context, items []Item) error
BatchGetItems(ctx context.Context, itemIds []string) ([]Item, error)
DeleteItem(ctx context.Context, itemId string) error
GetItem(ctx context.Context, itemId string) (Item, error)
ModifyItem(ctx context.Context, itemId string, patch ItemPatch) error
BatchGetItems(ctx context.Context, namespace string, itemIds []string) ([]Item, error)
DeleteItem(ctx context.Context, namespace, itemId string) error
GetItem(ctx context.Context, namespace, itemId string) (Item, error)
ModifyItem(ctx context.Context, namespace, itemId string, patch ItemPatch) error
GetItems(ctx context.Context, cursor string, n int, beginTime *time.Time) (string, []Item, error)
GetItemFeedback(ctx context.Context, itemId string, feedbackTypes ...string) ([]Feedback, error)
GetItemFeedback(ctx context.Context, namespace, itemId string, feedbackTypes ...string) ([]Feedback, error)
BatchInsertUsers(ctx context.Context, users []User) error
DeleteUser(ctx context.Context, userId string) error
GetUser(ctx context.Context, userId string) (User, error)
ModifyUser(ctx context.Context, userId string, patch UserPatch) error
GetUsers(ctx context.Context, cursor string, n int) (string, []User, error)
GetUserFeedback(ctx context.Context, userId string, endTime *time.Time, feedbackTypes ...string) ([]Feedback, error)
GetUserItemFeedback(ctx context.Context, userId, itemId string, feedbackTypes ...string) ([]Feedback, error)
DeleteUserItemFeedback(ctx context.Context, userId, itemId string, feedbackTypes ...string) (int, error)
GetUserFeedback(ctx context.Context, namespace, userId string, endTime *time.Time, feedbackTypes ...string) ([]Feedback, error)
GetUserItemFeedback(ctx context.Context, namespace, userId, itemId string, feedbackTypes ...string) ([]Feedback, error)
DeleteUserItemFeedback(ctx context.Context, namespace, userId, itemId string, feedbackTypes ...string) (int, error)
BatchInsertFeedback(ctx context.Context, feedback []Feedback, insertUser, insertItem, overwrite bool) error
GetFeedback(ctx context.Context, cursor string, n int, beginTime, endTime *time.Time, feedbackTypes ...string) (string, []Feedback, error)
GetUserStream(ctx context.Context, batchSize int) (chan []User, chan error)
Expand Down
Loading

0 comments on commit 6d3444e

Please sign in to comment.