Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cache last commit to accelerate the repository directory page visit #10069

Merged
merged 10 commits into from
Feb 1, 2020
14 changes: 13 additions & 1 deletion custom/conf/app.ini.sample
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ DEFAULT_CLOSE_ISSUES_VIA_COMMITS_IN_ANY_BRANCH = false
ENABLE_PUSH_CREATE_USER = false
ENABLE_PUSH_CREATE_ORG = false
; Comma separated list of globally disabled repo units. Allowed values: repo.issues, repo.ext_issues, repo.pulls, repo.wiki, repo.ext_wiki
DISABLED_REPO_UNITS =
DISABLED_REPO_UNITS =
; Comma separated list of default repo units. Allowed values: repo.code, repo.releases, repo.issues, repo.pulls, repo.wiki.
; Note: Code and Releases can currently not be deactivated. If you specify default repo units you should still list them for future compatibility.
; External wiki and issue tracker can't be enabled by default as it requires additional settings.
Expand Down Expand Up @@ -632,6 +632,8 @@ SENDMAIL_PATH = sendmail
SENDMAIL_ARGS =

[cache]
; if the cache enabled
ENABLED = true
; Either "memory", "redis", or "memcache", default is "memory"
ADAPTER = memory
; For "memory" only, GC interval in seconds, default is 60
Expand All @@ -644,6 +646,16 @@ HOST =
; Setting it to 0 disables caching
ITEM_TTL = 16h

; Last commit cache
[cache.last_commit]
; if the cache enabled
ENABLED = true
; Time to keep items in cache if not used, default is 8760 hours.
; Setting it to 0 disables caching
ITEM_TTL = 8760h
; Only enable the cache when repository's commits count great than
COMMITS_COUNT = 1000

[session]
; Either "memory", "file", or "redis", default is "memory"
PROVIDER = memory
Expand Down
7 changes: 7 additions & 0 deletions docs/content/doc/advanced/config-cheat-sheet.en-us.md
Original file line number Diff line number Diff line change
Expand Up @@ -383,13 +383,20 @@ relation to port exhaustion.

## Cache (`cache`)

- `ENABLED`: **true**: Enable the cache.
- `ADAPTER`: **memory**: Cache engine adapter, either `memory`, `redis`, or `memcache`.
- `INTERVAL`: **60**: Garbage Collection interval (sec), for memory cache only.
- `HOST`: **\<empty\>**: Connection string for `redis` and `memcache`.
- Redis: `network=tcp,addr=127.0.0.1:6379,password=macaron,db=0,pool_size=100,idle_timeout=180`
- Memcache: `127.0.0.1:9090;127.0.0.1:9091`
- `ITEM_TTL`: **16h**: Time to keep items in cache if not used, Setting it to 0 disables caching.

## Cache - LastCommitCache settings (`cache.last_commit`)

- `ENABLED`: **true**: Enable the cache.
- `ITEM_TTL`: **8760h**: Time to keep items in cache if not used, Setting it to 0 disables caching.
- `COMMITS_COUNT`: **1000**: Only enable the cache when repository's commits count great than.

## Session (`session`)

- `PROVIDER`: **memory**: Session engine provider \[memory, file, redis, mysql, couchbase, memcache, nodb, postgres\].
Expand Down
7 changes: 7 additions & 0 deletions docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
Original file line number Diff line number Diff line change
Expand Up @@ -148,13 +148,20 @@ menu:

## Cache (`cache`)

- `ENABLED`: **true**: 是否启用。
- `ADAPTER`: **memory**: 缓存引擎,可以为 `memory`, `redis` 或 `memcache`。
- `INTERVAL`: **60**: 只对内存缓存有效,GC间隔,单位秒。
- `HOST`: **\<empty\>**: 针对redis和memcache有效,主机地址和端口。
- Redis: `network=tcp,addr=127.0.0.1:6379,password=macaron,db=0,pool_size=100,idle_timeout=180`
- Memache: `127.0.0.1:9090;127.0.0.1:9091`
- `ITEM_TTL`: **16h**: 缓存项目失效时间,设置为 0 则禁用缓存。

## Cache - LastCommitCache settings (`cache.last_commit`)

- `ENABLED`: **true**: 是否启用。
- `ITEM_TTL`: **8760h**: 缓存项目失效时间,设置为 0 则禁用缓存。
- `COMMITS_COUNT`: **1000**: 仅当仓库的提交数大于时才启用缓存。

## Session (`session`)

- `PROVIDER`: Session 内容存储方式,可选 `memory`, `file`, `redis` 或 `mysql`。
Expand Down
65 changes: 63 additions & 2 deletions integrations/repo_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ package integrations
import (
"fmt"
"net/http"
"path"
"strings"
"testing"
"time"

"code.gitea.io/gitea/modules/setting"

Expand All @@ -29,12 +31,71 @@ func TestViewRepo(t *testing.T) {
session.MakeRequest(t, req, http.StatusNotFound)
}

func TestViewRepo2(t *testing.T) {
func testViewRepo(t *testing.T) {
defer prepareTestEnv(t)()

req := NewRequest(t, "GET", "/user3/repo3")
session := loginUser(t, "user2")
session.MakeRequest(t, req, http.StatusOK)
resp := session.MakeRequest(t, req, http.StatusOK)

htmlDoc := NewHTMLParser(t, resp.Body)
files := htmlDoc.doc.Find("#repo-files-table > TBODY > TR")

type file struct {
fileName string
commitID string
commitMsg string
commitTime string
}

var items []file

files.Each(func(i int, s *goquery.Selection) {
tds := s.Find("td")
var f file
tds.Each(func(i int, s *goquery.Selection) {
if i == 0 {
f.fileName = strings.TrimSpace(s.Text())
} else if i == 1 {
a := s.Find("a")
f.commitMsg = strings.TrimSpace(a.Text())
l, _ := a.Attr("href")
f.commitID = path.Base(l)
}
})

f.commitTime, _ = s.Find("span.time-since").Attr("title")
items = append(items, f)
})

assert.EqualValues(t, []file{
{
fileName: "doc",
commitID: "2a47ca4b614a9f5a43abbd5ad851a54a616ffee6",
commitMsg: "init project",
commitTime: time.Date(2017, time.June, 14, 13, 54, 21, 0, time.UTC).Format(time.RFC1123),
},
{
fileName: "README.md",
commitID: "2a47ca4b614a9f5a43abbd5ad851a54a616ffee6",
commitMsg: "init project",
commitTime: time.Date(2017, time.June, 14, 13, 54, 21, 0, time.UTC).Format(time.RFC1123),
},
}, items)
}

func TestViewRepo2(t *testing.T) {
// no last commit cache
testViewRepo(t)

// enable last commit cache for all repositories
oldCommitsCount := setting.CacheService.LastCommit.CommitsCount
setting.CacheService.LastCommit.CommitsCount = 0
// first view will not hit the cache
testViewRepo(t)
// second view will hit the cache
testViewRepo(t)
setting.CacheService.LastCommit.CommitsCount = oldCommitsCount
}

func TestViewRepo3(t *testing.T) {
Expand Down
26 changes: 17 additions & 9 deletions modules/cache/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,28 @@ import (
_ "gitea.com/macaron/cache/redis"
)

var conn mc.Cache
var (
conn mc.Cache
)

func newCache(cacheConfig setting.Cache) (mc.Cache, error) {
return mc.NewCacher(cacheConfig.Adapter, mc.Options{
Adapter: cacheConfig.Adapter,
AdapterConfig: cacheConfig.Conn,
Interval: cacheConfig.Interval,
})
}

// NewContext start cache service
func NewContext() error {
if setting.CacheService == nil || conn != nil {
return nil
var err error

if conn == nil && setting.CacheService.Enabled {
if conn, err = newCache(setting.CacheService.Cache); err != nil {
return err
}
}

var err error
conn, err = mc.NewCacher(setting.CacheService.Adapter, mc.Options{
Adapter: setting.CacheService.Adapter,
AdapterConfig: setting.CacheService.Conn,
Interval: setting.CacheService.Interval,
})
return err
}

Expand Down
64 changes: 64 additions & 0 deletions modules/cache/last_commit.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package cache

import (
"fmt"

"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"

mc "gitea.com/macaron/cache"
"gopkg.in/src-d/go-git.v4/plumbing/object"
)

// LastCommitCache represents a cache to store last commit
type LastCommitCache struct {
repoPath string
ttl int64
repo *git.Repository
commitCache map[string]*object.Commit
mc.Cache
}

// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(repoPath string, gitRepo *git.Repository, ttl int64) *LastCommitCache {
return &LastCommitCache{
repoPath: repoPath,
repo: gitRepo,
commitCache: make(map[string]*object.Commit),
ttl: ttl,
Cache: conn,
}
}

// Get get the last commit information by commit id and entry path
func (c LastCommitCache) Get(ref, entryPath string) (*object.Commit, error) {
v := c.Cache.Get(fmt.Sprintf("last_commit:%s:%s:%s", c.repoPath, ref, entryPath))
if vs, ok := v.(string); ok {
log.Trace("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
if commit, ok := c.commitCache[vs]; ok {
log.Trace("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
return commit, nil
}
id, err := c.repo.ConvertToSHA1(vs)
if err != nil {
return nil, err
}
commit, err := c.repo.GoGitRepo().CommitObject(id)
if err != nil {
return nil, err
}
c.commitCache[vs] = commit
return commit, nil
}
return nil, nil
}

// Put put the last commit id with commit and entry path
func (c LastCommitCache) Put(ref, entryPath, commitID string) error {
log.Trace("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
return c.Cache.Put(fmt.Sprintf("last_commit:%s:%s:%s", c.repoPath, ref, entryPath), commitID, c.ttl)
}
6 changes: 4 additions & 2 deletions modules/git/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@

package git

import "gopkg.in/src-d/go-git.v4/plumbing/object"

// LastCommitCache cache
type LastCommitCache interface {
Get(repoPath, ref, entryPath string) (*Commit, error)
Put(repoPath, ref, entryPath string, commit *Commit) error
Get(ref, entryPath string) (*object.Commit, error)
Put(ref, entryPath, commitID string) error
}
45 changes: 44 additions & 1 deletion modules/git/commit_info.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
package git

import (
"path"

"github.com/emirpasic/gods/trees/binaryheap"
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/plumbing/object"
Expand All @@ -30,7 +32,29 @@ func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCom
return nil, nil, err
}

revs, err := getLastCommitForPaths(c, treePath, entryPaths)
var revs map[string]*object.Commit
if cache != nil {
var unHitPaths []string
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
if err != nil {
return nil, nil, err
}
if len(unHitPaths) > 0 {
revs2, err := getLastCommitForPaths(c, treePath, unHitPaths)
if err != nil {
return nil, nil, err
}

for k, v := range revs2 {
if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil {
return nil, nil, err
}
revs[k] = v
}
}
} else {
revs, err = getLastCommitForPaths(c, treePath, entryPaths)
}
if err != nil {
return nil, nil, err
}
Expand Down Expand Up @@ -127,6 +151,25 @@ func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[
return hashes, nil
}

func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache LastCommitCache) (map[string]*object.Commit, []string, error) {
var unHitEntryPaths []string
var results = make(map[string]*object.Commit)
for _, p := range paths {
lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
if err != nil {
return nil, nil, err
}
if lastCommit != nil {
results[p] = lastCommit
continue
}

unHitEntryPaths = append(unHitEntryPaths, p)
}

return results, unHitEntryPaths, nil
}

func getLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
// We do a tree traversal with nodes sorted by commit time
heap := binaryheap.NewWith(func(a, b interface{}) int {
Expand Down
Loading