Michael Muré created
api/graphql/resolvers/repo.go | 5 +
cache/repo_cache.go | 14 ++---
cache/repo_cache_bug.go | 60 +++++++++---------------
cache/repo_cache_test.go | 4 +
commands/ls.go | 5 +
identity/identity_test.go | 4
repository/git.go | 86 +++++++++++++++++++++++++++++++++---
repository/gogit.go | 76 ++++++++++++++++++++++++++++++++
repository/mock_repo.go | 47 ++++++++++++++++++++
repository/repo.go | 14 ++++++
termui/bug_table.go | 6 ++
11 files changed, 262 insertions(+), 59 deletions(-)
@@ -41,7 +41,10 @@ func (repoResolver) AllBugs(_ context.Context, obj *models.Repository, after *st
}
// Simply pass a []string with the ids to the pagination algorithm
- source := obj.Repo.QueryBugs(q)
+ source, err := obj.Repo.QueryBugs(q)
+ if err != nil {
+ return nil, err
+ }
// The edger create a custom edge holding just the id
edger := func(id entity.Id, offset int) connections.Edge {
@@ -13,7 +13,6 @@ import (
"github.com/MichaelMure/git-bug/identity"
"github.com/MichaelMure/git-bug/repository"
"github.com/MichaelMure/git-bug/util/process"
- "github.com/blevesearch/bleve"
)
// 1: original format
@@ -55,8 +54,6 @@ type RepoCache struct {
muBug sync.RWMutex
// excerpt of bugs data for all bugs
bugExcerpts map[entity.Id]*BugExcerpt
- // searchable cache of all bugs
- searchCache bleve.Index
// bug loaded in memory
bugs map[entity.Id]*BugCache
// loadedBugs is an LRU cache that records which bugs the cache has loaded in
@@ -161,9 +158,9 @@ func (c *RepoCache) Close() error {
c.bugs = make(map[entity.Id]*BugCache)
c.bugExcerpts = nil
- if c.searchCache != nil {
- c.searchCache.Close()
- c.searchCache = nil
+ err := c.repo.Close()
+ if err != nil {
+ return err
}
return c.repo.LocalStorage().Remove(lockfile)
@@ -199,9 +196,10 @@ func (c *RepoCache) buildCache() error {
allBugs := bug.ReadAllLocal(c.repo)
- err := c.createBleveIndex()
+ // wipe the index just to be sure
+ err := c.repo.ClearBleveIndex("bug")
if err != nil {
- return fmt.Errorf("Unable to create search cache. Error: %v", err)
+ return err
}
for b := range allBugs {
@@ -23,10 +23,6 @@ const (
var errBugNotInCache = errors.New("bug missing from cache")
-func searchCacheDirPath(repo repository.Repo) string {
- return path.Join(repo.GetPath(), "git-bug", searchCacheDir)
-}
-
// bugUpdated is a callback to trigger when the excerpt of a bug changed,
// that is each time a bug is updated
func (c *RepoCache) bugUpdated(id entity.Id) error {
@@ -82,14 +78,13 @@ func (c *RepoCache) loadBugCache() error {
c.bugExcerpts = aux.Excerpts
- blevePath := searchCacheDirPath(c.repo)
- searchCache, err := bleve.Open(blevePath)
+ index, err := c.repo.GetBleveIndex("bug")
if err != nil {
- return fmt.Errorf("Unable to open search cache. Error: %v", err)
+ return err
}
- c.searchCache = searchCache
- count, err := c.searchCache.DocCount()
+ // simple heuristic to detect a mismatch between the index and the bugs
+ count, err := index.DocCount()
if err != nil {
return err
}
@@ -100,26 +95,6 @@ func (c *RepoCache) loadBugCache() error {
return nil
}
-func (c *RepoCache) createBleveIndex() error {
- blevePath := searchCacheDirPath(c.repo)
-
- _ = os.RemoveAll(blevePath)
-
- mapping := bleve.NewIndexMapping()
- mapping.DefaultAnalyzer = "en"
-
- dir := searchCacheDirPath(c.repo)
-
- bleveIndex, err := bleve.New(dir, mapping)
- if err != nil {
- return err
- }
-
- c.searchCache = bleveIndex
-
- return nil
-}
-
// write will serialize on disk the bug cache file
func (c *RepoCache) writeBugCache() error {
c.muBug.RLock()
@@ -287,12 +262,12 @@ func (c *RepoCache) resolveBugMatcher(f func(*BugExcerpt) bool) (entity.Id, erro
}
// QueryBugs return the id of all Bug matching the given Query
-func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id {
+func (c *RepoCache) QueryBugs(q *query.Query) ([]entity.Id, error) {
c.muBug.RLock()
defer c.muBug.RUnlock()
if q == nil {
- return c.AllBugsIds()
+ return c.AllBugsIds(), nil
}
matcher := compileMatcher(q.Filters)
@@ -313,9 +288,15 @@ func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id {
bleveQuery := bleve.NewQueryStringQuery(strings.Join(terms, " "))
bleveSearch := bleve.NewSearchRequest(bleveQuery)
- searchResults, err := c.searchCache.Search(bleveSearch)
+
+ index, err := c.repo.GetBleveIndex("bug")
+ if err != nil {
+ return nil, err
+ }
+
+ searchResults, err := index.Search(bleveSearch)
if err != nil {
- panic("bleve search failed")
+ return nil, err
}
for _, hit := range searchResults.Hits {
@@ -341,7 +322,7 @@ func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id {
case query.OrderByEdit:
sorter = BugsByEditTime(filtered)
default:
- panic("missing sort type")
+ return nil, errors.New("missing sort type")
}
switch q.OrderDirection {
@@ -350,7 +331,7 @@ func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id {
case query.OrderDescending:
sorter = sort.Reverse(sorter)
default:
- panic("missing sort direction")
+ return nil, errors.New("missing sort direction")
}
sort.Sort(sorter)
@@ -361,7 +342,7 @@ func (c *RepoCache) QueryBugs(q *query.Query) []entity.Id {
result[i] = val.Id
}
- return result
+ return result, nil
}
// AllBugsIds return all known bug ids
@@ -504,7 +485,12 @@ func (c *RepoCache) addBugToSearchIndex(snap *bug.Snapshot) error {
searchableBug.Text = append(searchableBug.Text, snap.Title)
- err := c.searchCache.Index(snap.Id().String(), searchableBug)
+ index, err := c.repo.GetBleveIndex("bug")
+ if err != nil {
+ return err
+ }
+
+ err = index.Index(snap.Id().String(), searchableBug)
if err != nil {
return err
}
@@ -73,7 +73,9 @@ func TestCache(t *testing.T) {
// Querying
q, err := query.Parse("status:open author:descartes sort:edit-asc")
require.NoError(t, err)
- require.Len(t, cache.QueryBugs(q), 2)
+ res, err := cache.QueryBugs(q)
+ require.NoError(t, err)
+ require.Len(t, res, 2)
// Close
require.NoError(t, cache.Close())
@@ -110,7 +110,10 @@ func runLs(env *Env, opts lsOptions, args []string) error {
return err
}
- allIds := env.backend.QueryBugs(q)
+ allIds, err := env.backend.QueryBugs(q)
+ if err != nil {
+ return err
+ }
bugExcerpt := make([]*cache.BugExcerpt, len(allIds))
for i, id := range allIds {
@@ -273,10 +273,10 @@ func TestIdentityRemove(t *testing.T) {
remoteB := repository.CreateGoGitTestRepo(true)
defer repository.CleanupTestRepos(repo, remoteA, remoteB)
- err := repo.AddRemote("remoteA", "file://"+remoteA.GetPath())
+ err := repo.AddRemote("remoteA", remoteA.GetLocalRemote())
require.NoError(t, err)
- err = repo.AddRemote("remoteB", "file://"+remoteB.GetPath())
+ err = repo.AddRemote("remoteB", remoteB.GetLocalRemote())
require.NoError(t, err)
// generate an identity for testing
@@ -9,6 +9,7 @@ import (
"strings"
"sync"
+ "github.com/blevesearch/bleve"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/osfs"
@@ -30,12 +31,15 @@ type GitRepo struct {
clocksMutex sync.Mutex
clocks map[string]lamport.Clock
+ indexesMutex sync.Mutex
+ indexes map[string]bleve.Index
+
keyring Keyring
}
-// NewGitRepo determines if the given working directory is inside of a git repository,
+// OpenGitRepo determines if the given working directory is inside of a git repository,
// and returns the corresponding GitRepo instance if it is.
-func NewGitRepo(path string, clockLoaders []ClockLoader) (*GitRepo, error) {
+func OpenGitRepo(path string, clockLoaders []ClockLoader) (*GitRepo, error) {
k, err := defaultKeyring()
if err != nil {
return nil, err
@@ -45,6 +49,7 @@ func NewGitRepo(path string, clockLoaders []ClockLoader) (*GitRepo, error) {
gitCli: gitCli{path: path},
path: path,
clocks: make(map[string]lamport.Clock),
+ indexes: make(map[string]bleve.Index),
keyring: k,
}
@@ -84,9 +89,10 @@ func NewGitRepo(path string, clockLoaders []ClockLoader) (*GitRepo, error) {
// InitGitRepo create a new empty git repo at the given path
func InitGitRepo(path string) (*GitRepo, error) {
repo := &GitRepo{
- gitCli: gitCli{path: path},
- path: path + "/.git",
- clocks: make(map[string]lamport.Clock),
+ gitCli: gitCli{path: path},
+ path: path + "/.git",
+ clocks: make(map[string]lamport.Clock),
+ indexes: make(map[string]bleve.Index),
}
_, err := repo.runGitCommand("init", path)
@@ -100,9 +106,10 @@ func InitGitRepo(path string) (*GitRepo, error) {
// InitBareGitRepo create a new --bare empty git repo at the given path
func InitBareGitRepo(path string) (*GitRepo, error) {
repo := &GitRepo{
- gitCli: gitCli{path: path},
- path: path,
- clocks: make(map[string]lamport.Clock),
+ gitCli: gitCli{path: path},
+ path: path,
+ clocks: make(map[string]lamport.Clock),
+ indexes: make(map[string]bleve.Index),
}
_, err := repo.runGitCommand("init", "--bare", path)
@@ -113,6 +120,17 @@ func InitBareGitRepo(path string) (*GitRepo, error) {
return repo, nil
}
+func (repo *GitRepo) Close() error {
+ var firstErr error
+ for _, index := range repo.indexes {
+ err := index.Close()
+ if err != nil && firstErr == nil {
+ firstErr = err
+ }
+ }
+ return firstErr
+}
+
// LocalConfig give access to the repository scoped configuration
func (repo *GitRepo) LocalConfig() Config {
return newGitConfig(repo.gitCli, false)
@@ -183,6 +201,58 @@ func (repo *GitRepo) LocalStorage() billy.Filesystem {
return osfs.New(repo.path)
}
+// GetBleveIndex return a bleve.Index that can be used to index documents
+func (repo *GitRepo) GetBleveIndex(name string) (bleve.Index, error) {
+ repo.indexesMutex.Lock()
+ defer repo.indexesMutex.Unlock()
+
+ if index, ok := repo.indexes[name]; ok {
+ return index, nil
+ }
+
+ path := filepath.Join(repo.path, "indexes", name)
+
+ index, err := bleve.Open(path)
+ if err == nil {
+ repo.indexes[name] = index
+ return index, nil
+ }
+
+ err = os.MkdirAll(path, os.ModeDir)
+ if err != nil {
+ return nil, err
+ }
+
+ mapping := bleve.NewIndexMapping()
+ mapping.DefaultAnalyzer = "en"
+
+ index, err = bleve.New(path, mapping)
+ if err != nil {
+ return nil, err
+ }
+
+ repo.indexes[name] = index
+
+ return index, nil
+}
+
+// ClearBleveIndex will wipe the given index
+func (repo *GitRepo) ClearBleveIndex(name string) error {
+ repo.indexesMutex.Lock()
+ defer repo.indexesMutex.Unlock()
+
+ path := filepath.Join(repo.path, "indexes", name)
+
+ err := os.RemoveAll(path)
+ if err != nil {
+ return err
+ }
+
+ delete(repo.indexes, name)
+
+ return nil
+}
+
// FetchRefs fetch git refs from a remote
func (repo *GitRepo) FetchRefs(remote, refSpec string) (string, error) {
stdout, err := repo.runGitCommand("fetch", remote, refSpec)
@@ -12,6 +12,7 @@ import (
"sync"
"time"
+ "github.com/blevesearch/bleve"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/osfs"
gogit "github.com/go-git/go-git/v5"
@@ -33,6 +34,9 @@ type GoGitRepo struct {
clocksMutex sync.Mutex
clocks map[string]lamport.Clock
+ indexesMutex sync.Mutex
+ indexes map[string]bleve.Index
+
keyring Keyring
localStorage billy.Filesystem
}
@@ -58,6 +62,7 @@ func OpenGoGitRepo(path string, clockLoaders []ClockLoader) (*GoGitRepo, error)
r: r,
path: path,
clocks: make(map[string]lamport.Clock),
+ indexes: make(map[string]bleve.Index),
keyring: k,
localStorage: osfs.New(filepath.Join(path, "git-bug")),
}
@@ -97,6 +102,7 @@ func InitGoGitRepo(path string) (*GoGitRepo, error) {
r: r,
path: filepath.Join(path, ".git"),
clocks: make(map[string]lamport.Clock),
+ indexes: make(map[string]bleve.Index),
keyring: k,
localStorage: osfs.New(filepath.Join(path, ".git", "git-bug")),
}, nil
@@ -118,6 +124,7 @@ func InitBareGoGitRepo(path string) (*GoGitRepo, error) {
r: r,
path: path,
clocks: make(map[string]lamport.Clock),
+ indexes: make(map[string]bleve.Index),
keyring: k,
localStorage: osfs.New(filepath.Join(path, "git-bug")),
}, nil
@@ -179,6 +186,17 @@ func isGitDir(path string) (bool, error) {
return true, nil
}
+func (repo *GoGitRepo) Close() error {
+ var firstErr error
+ for _, index := range repo.indexes {
+ err := index.Close()
+ if err != nil && firstErr == nil {
+ firstErr = err
+ }
+ }
+ return firstErr
+}
+
// LocalConfig give access to the repository scoped configuration
func (repo *GoGitRepo) LocalConfig() Config {
return newGoGitLocalConfig(repo.r)
@@ -274,6 +292,64 @@ func (repo *GoGitRepo) LocalStorage() billy.Filesystem {
return repo.localStorage
}
+// GetBleveIndex return a bleve.Index that can be used to index documents
+func (repo *GoGitRepo) GetBleveIndex(name string) (bleve.Index, error) {
+ repo.indexesMutex.Lock()
+ defer repo.indexesMutex.Unlock()
+
+ if index, ok := repo.indexes[name]; ok {
+ return index, nil
+ }
+
+ path := filepath.Join(repo.path, "git-bug", "indexes", name)
+
+ index, err := bleve.Open(path)
+ if err == nil {
+ repo.indexes[name] = index
+ return index, nil
+ }
+
+ err = os.MkdirAll(path, os.ModePerm)
+ if err != nil {
+ return nil, err
+ }
+
+ mapping := bleve.NewIndexMapping()
+ mapping.DefaultAnalyzer = "en"
+
+ index, err = bleve.New(path, mapping)
+ if err != nil {
+ return nil, err
+ }
+
+ repo.indexes[name] = index
+
+ return index, nil
+}
+
+// ClearBleveIndex will wipe the given index
+func (repo *GoGitRepo) ClearBleveIndex(name string) error {
+ repo.indexesMutex.Lock()
+ defer repo.indexesMutex.Unlock()
+
+ path := filepath.Join(repo.path, "indexes", name)
+
+ err := os.RemoveAll(path)
+ if err != nil {
+ return err
+ }
+
+ if index, ok := repo.indexes[name]; ok {
+ err = index.Close()
+ if err != nil {
+ return err
+ }
+ delete(repo.indexes, name)
+ }
+
+ return nil
+}
+
// FetchRefs fetch git refs from a remote
func (repo *GoGitRepo) FetchRefs(remote string, refSpec string) (string, error) {
buf := bytes.NewBuffer(nil)
@@ -7,6 +7,7 @@ import (
"sync"
"github.com/99designs/keyring"
+ "github.com/blevesearch/bleve"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
@@ -22,16 +23,20 @@ type mockRepoForTest struct {
*mockRepoKeyring
*mockRepoCommon
*mockRepoStorage
+ *mockRepoBleve
*mockRepoData
*mockRepoClock
}
+func (m *mockRepoForTest) Close() error { return nil }
+
func NewMockRepoForTest() *mockRepoForTest {
return &mockRepoForTest{
mockRepoConfig: NewMockRepoConfig(),
mockRepoKeyring: NewMockRepoKeyring(),
mockRepoCommon: NewMockRepoCommon(),
mockRepoStorage: NewMockRepoStorage(),
+ mockRepoBleve: newMockRepoBleve(),
mockRepoData: NewMockRepoData(),
mockRepoClock: NewMockRepoClock(),
}
@@ -126,6 +131,48 @@ func (m *mockRepoStorage) LocalStorage() billy.Filesystem {
return m.localFs
}
+var _ RepoBleve = &mockRepoBleve{}
+
+type mockRepoBleve struct {
+ indexesMutex sync.Mutex
+ indexes map[string]bleve.Index
+}
+
+func newMockRepoBleve() *mockRepoBleve {
+ return &mockRepoBleve{
+ indexes: make(map[string]bleve.Index),
+ }
+}
+
+func (m *mockRepoBleve) GetBleveIndex(name string) (bleve.Index, error) {
+ m.indexesMutex.Lock()
+ defer m.indexesMutex.Unlock()
+
+ if index, ok := m.indexes[name]; ok {
+ return index, nil
+ }
+
+ mapping := bleve.NewIndexMapping()
+ mapping.DefaultAnalyzer = "en"
+
+ index, err := bleve.NewMemOnly(mapping)
+ if err != nil {
+ return nil, err
+ }
+
+ m.indexes[name] = index
+
+ return index, nil
+}
+
+func (m *mockRepoBleve) ClearBleveIndex(name string) error {
+ m.indexesMutex.Lock()
+ defer m.indexesMutex.Unlock()
+
+ delete(m.indexes, name)
+ return nil
+}
+
var _ RepoData = &mockRepoData{}
type commit struct {
@@ -4,6 +4,7 @@ package repository
import (
"errors"
+ "github.com/blevesearch/bleve"
"github.com/go-git/go-billy/v5"
"github.com/MichaelMure/git-bug/util/lamport"
@@ -23,6 +24,9 @@ type Repo interface {
RepoCommon
RepoData
RepoStorage
+ RepoBleve
+
+ Close() error
}
type RepoCommonStorage interface {
@@ -69,11 +73,21 @@ type RepoCommon interface {
GetRemotes() (map[string]string, error)
}
+// RepoStorage give access to the filesystem
type RepoStorage interface {
// LocalStorage return a billy.Filesystem giving access to $RepoPath/.git/git-bug
LocalStorage() billy.Filesystem
}
+// RepoBleve give access to Bleve to implement full-text search indexes.
+type RepoBleve interface {
+ // GetBleveIndex return a bleve.Index that can be used to index documents
+ GetBleveIndex(name string) (bleve.Index, error)
+
+ // ClearBleveIndex will wipe the given index
+ ClearBleveIndex(name string) error
+}
+
// RepoData give access to the git data storage
type RepoData interface {
// FetchRefs fetch git refs from a remote
@@ -237,7 +237,11 @@ func (bt *bugTable) disable(g *gocui.Gui) error {
}
func (bt *bugTable) paginate(max int) error {
- bt.allIds = bt.repo.QueryBugs(bt.query)
+ var err error
+ bt.allIds, err = bt.repo.QueryBugs(bt.query)
+ if err != nil {
+ return err
+ }
return bt.doPaginate(max)
}