1package cache
2
3import (
4 "bytes"
5 "encoding/gob"
6 "errors"
7 "fmt"
8 "sort"
9 "strings"
10 "time"
11 "unicode/utf8"
12
13 "github.com/blevesearch/bleve"
14
15 "github.com/MichaelMure/git-bug/bug"
16 "github.com/MichaelMure/git-bug/entity"
17 "github.com/MichaelMure/git-bug/query"
18 "github.com/MichaelMure/git-bug/repository"
19)
20
21const (
22 bugCacheFile = "bug-cache"
23 searchCacheDir = "search-cache"
24)
25
26var errBugNotInCache = errors.New("bug missing from cache")
27
28// bugUpdated is a callback to trigger when the excerpt of a bug changed,
29// that is each time a bug is updated
30func (c *RepoCache) bugUpdated(id entity.Id) error {
31 c.muBug.Lock()
32 b, ok := c.bugs[id]
33 if !ok {
34 c.muBug.Unlock()
35
36 // if the bug is not loaded at this point, it means it was loaded before
37 // but got evicted. Which means we potentially have multiple copies in
38 // memory and thus concurrent write.
39 // Failing immediately here is the simple and safe solution to avoid
40 // complicated data loss.
41 return errBugNotInCache
42 }
43 c.loadedBugs.Get(id)
44 c.bugExcerpts[id] = NewBugExcerpt(b.bug, b.Snapshot())
45 c.muBug.Unlock()
46
47 if err := c.addBugToSearchIndex(b.Snapshot()); err != nil {
48 return err
49 }
50
51 // we only need to write the bug cache
52 return c.writeBugCache()
53}
54
55// load will try to read from the disk the bug cache file
56func (c *RepoCache) loadBugCache() error {
57 c.muBug.Lock()
58 defer c.muBug.Unlock()
59
60 f, err := c.repo.LocalStorage().Open(bugCacheFile)
61 if err != nil {
62 return err
63 }
64
65 decoder := gob.NewDecoder(f)
66
67 aux := struct {
68 Version uint
69 Excerpts map[entity.Id]*BugExcerpt
70 }{}
71
72 err = decoder.Decode(&aux)
73 if err != nil {
74 return err
75 }
76
77 if aux.Version != formatVersion {
78 return fmt.Errorf("unknown cache format version %v", aux.Version)
79 }
80
81 c.bugExcerpts = aux.Excerpts
82
83 index, err := c.repo.GetBleveIndex("bug")
84 if err != nil {
85 return err
86 }
87
88 // simple heuristic to detect a mismatch between the index and the bugs
89 count, err := index.DocCount()
90 if err != nil {
91 return err
92 }
93 if count != uint64(len(c.bugExcerpts)) {
94 return fmt.Errorf("count mismatch between bleve and bug excerpts")
95 }
96
97 return nil
98}
99
100// write will serialize on disk the bug cache file
101func (c *RepoCache) writeBugCache() error {
102 c.muBug.RLock()
103 defer c.muBug.RUnlock()
104
105 var data bytes.Buffer
106
107 aux := struct {
108 Version uint
109 Excerpts map[entity.Id]*BugExcerpt
110 }{
111 Version: formatVersion,
112 Excerpts: c.bugExcerpts,
113 }
114
115 encoder := gob.NewEncoder(&data)
116
117 err := encoder.Encode(aux)
118 if err != nil {
119 return err
120 }
121
122 f, err := c.repo.LocalStorage().Create(bugCacheFile)
123 if err != nil {
124 return err
125 }
126
127 _, err = f.Write(data.Bytes())
128 if err != nil {
129 return err
130 }
131
132 return f.Close()
133}
134
135// ResolveBugExcerpt retrieve a BugExcerpt matching the exact given id
136func (c *RepoCache) ResolveBugExcerpt(id entity.Id) (*BugExcerpt, error) {
137 c.muBug.RLock()
138 defer c.muBug.RUnlock()
139
140 excerpt, ok := c.bugExcerpts[id]
141 if !ok {
142 return nil, bug.ErrBugNotExist
143 }
144
145 return excerpt, nil
146}
147
148// ResolveBug retrieve a bug matching the exact given id
149func (c *RepoCache) ResolveBug(id entity.Id) (*BugCache, error) {
150 c.muBug.RLock()
151 cached, ok := c.bugs[id]
152 if ok {
153 c.loadedBugs.Get(id)
154 c.muBug.RUnlock()
155 return cached, nil
156 }
157 c.muBug.RUnlock()
158
159 b, err := bug.ReadLocalWithResolver(c.repo, newIdentityCacheResolver(c), id)
160 if err != nil {
161 return nil, err
162 }
163
164 cached = NewBugCache(c, b)
165
166 c.muBug.Lock()
167 c.bugs[id] = cached
168 c.loadedBugs.Add(id)
169 c.muBug.Unlock()
170
171 c.evictIfNeeded()
172
173 return cached, nil
174}
175
176// evictIfNeeded will evict a bug from the cache if needed
177// it also removes references of the bug from the bugs
178func (c *RepoCache) evictIfNeeded() {
179 c.muBug.Lock()
180 defer c.muBug.Unlock()
181 if c.loadedBugs.Len() <= c.maxLoadedBugs {
182 return
183 }
184
185 for _, id := range c.loadedBugs.GetOldestToNewest() {
186 b := c.bugs[id]
187 if b.NeedCommit() {
188 continue
189 }
190
191 b.mu.Lock()
192 c.loadedBugs.Remove(id)
193 delete(c.bugs, id)
194
195 if c.loadedBugs.Len() <= c.maxLoadedBugs {
196 return
197 }
198 }
199}
200
201// ResolveBugExcerptPrefix retrieve a BugExcerpt matching an id prefix. It fails if multiple
202// bugs match.
203func (c *RepoCache) ResolveBugExcerptPrefix(prefix string) (*BugExcerpt, error) {
204 return c.ResolveBugExcerptMatcher(func(excerpt *BugExcerpt) bool {
205 return excerpt.Id.HasPrefix(prefix)
206 })
207}
208
209// ResolveBugPrefix retrieve a bug matching an id prefix. It fails if multiple
210// bugs match.
211func (c *RepoCache) ResolveBugPrefix(prefix string) (*BugCache, error) {
212 return c.ResolveBugMatcher(func(excerpt *BugExcerpt) bool {
213 return excerpt.Id.HasPrefix(prefix)
214 })
215}
216
217// ResolveBugCreateMetadata retrieve a bug that has the exact given metadata on
218// its Create operation, that is, the first operation. It fails if multiple bugs
219// match.
220func (c *RepoCache) ResolveBugCreateMetadata(key string, value string) (*BugCache, error) {
221 return c.ResolveBugMatcher(func(excerpt *BugExcerpt) bool {
222 return excerpt.CreateMetadata[key] == value
223 })
224}
225
226func (c *RepoCache) ResolveBugExcerptMatcher(f func(*BugExcerpt) bool) (*BugExcerpt, error) {
227 id, err := c.resolveBugMatcher(f)
228 if err != nil {
229 return nil, err
230 }
231 return c.ResolveBugExcerpt(id)
232}
233
234func (c *RepoCache) ResolveBugMatcher(f func(*BugExcerpt) bool) (*BugCache, error) {
235 id, err := c.resolveBugMatcher(f)
236 if err != nil {
237 return nil, err
238 }
239 return c.ResolveBug(id)
240}
241
242func (c *RepoCache) resolveBugMatcher(f func(*BugExcerpt) bool) (entity.Id, error) {
243 c.muBug.RLock()
244 defer c.muBug.RUnlock()
245
246 // preallocate but empty
247 matching := make([]entity.Id, 0, 5)
248
249 for _, excerpt := range c.bugExcerpts {
250 if f(excerpt) {
251 matching = append(matching, excerpt.Id)
252 }
253 }
254
255 if len(matching) > 1 {
256 return entity.UnsetId, bug.NewErrMultipleMatchBug(matching)
257 }
258
259 if len(matching) == 0 {
260 return entity.UnsetId, bug.ErrBugNotExist
261 }
262
263 return matching[0], nil
264}
265
266// QueryBugs return the id of all Bug matching the given Query
267func (c *RepoCache) QueryBugs(q *query.Query) ([]entity.Id, error) {
268 c.muBug.RLock()
269 defer c.muBug.RUnlock()
270
271 if q == nil {
272 return c.AllBugsIds(), nil
273 }
274
275 matcher := compileMatcher(q.Filters)
276
277 var filtered []*BugExcerpt
278 var foundBySearch map[entity.Id]*BugExcerpt
279
280 if q.Search != nil {
281 foundBySearch = map[entity.Id]*BugExcerpt{}
282
283 terms := make([]string, len(q.Search))
284 copy(terms, q.Search)
285 for i, search := range q.Search {
286 if strings.Contains(search, " ") {
287 terms[i] = fmt.Sprintf("\"%s\"", search)
288 }
289 }
290
291 bleveQuery := bleve.NewQueryStringQuery(strings.Join(terms, " "))
292 bleveSearch := bleve.NewSearchRequest(bleveQuery)
293
294 index, err := c.repo.GetBleveIndex("bug")
295 if err != nil {
296 return nil, err
297 }
298
299 searchResults, err := index.Search(bleveSearch)
300 if err != nil {
301 return nil, err
302 }
303
304 for _, hit := range searchResults.Hits {
305 foundBySearch[entity.Id(hit.ID)] = c.bugExcerpts[entity.Id(hit.ID)]
306 }
307 } else {
308 foundBySearch = c.bugExcerpts
309 }
310
311 for _, excerpt := range foundBySearch {
312 if matcher.Match(excerpt, c) {
313 filtered = append(filtered, excerpt)
314 }
315 }
316
317 var sorter sort.Interface
318
319 switch q.OrderBy {
320 case query.OrderById:
321 sorter = BugsById(filtered)
322 case query.OrderByCreation:
323 sorter = BugsByCreationTime(filtered)
324 case query.OrderByEdit:
325 sorter = BugsByEditTime(filtered)
326 default:
327 return nil, errors.New("missing sort type")
328 }
329
330 switch q.OrderDirection {
331 case query.OrderAscending:
332 // Nothing to do
333 case query.OrderDescending:
334 sorter = sort.Reverse(sorter)
335 default:
336 return nil, errors.New("missing sort direction")
337 }
338
339 sort.Sort(sorter)
340
341 result := make([]entity.Id, len(filtered))
342
343 for i, val := range filtered {
344 result[i] = val.Id
345 }
346
347 return result, nil
348}
349
350// AllBugsIds return all known bug ids
351func (c *RepoCache) AllBugsIds() []entity.Id {
352 c.muBug.RLock()
353 defer c.muBug.RUnlock()
354
355 result := make([]entity.Id, len(c.bugExcerpts))
356
357 i := 0
358 for _, excerpt := range c.bugExcerpts {
359 result[i] = excerpt.Id
360 i++
361 }
362
363 return result
364}
365
366// ValidLabels list valid labels
367//
368// Note: in the future, a proper label policy could be implemented where valid
369// labels are defined in a configuration file. Until that, the default behavior
370// is to return the list of labels already used.
371func (c *RepoCache) ValidLabels() []bug.Label {
372 c.muBug.RLock()
373 defer c.muBug.RUnlock()
374
375 set := map[bug.Label]interface{}{}
376
377 for _, excerpt := range c.bugExcerpts {
378 for _, l := range excerpt.Labels {
379 set[l] = nil
380 }
381 }
382
383 result := make([]bug.Label, len(set))
384
385 i := 0
386 for l := range set {
387 result[i] = l
388 i++
389 }
390
391 // Sort
392 sort.Slice(result, func(i, j int) bool {
393 return string(result[i]) < string(result[j])
394 })
395
396 return result
397}
398
399// NewBug create a new bug
400// The new bug is written in the repository (commit)
401func (c *RepoCache) NewBug(title string, message string) (*BugCache, *bug.CreateOperation, error) {
402 return c.NewBugWithFiles(title, message, nil)
403}
404
405// NewBugWithFiles create a new bug with attached files for the message
406// The new bug is written in the repository (commit)
407func (c *RepoCache) NewBugWithFiles(title string, message string, files []repository.Hash) (*BugCache, *bug.CreateOperation, error) {
408 author, err := c.GetUserIdentity()
409 if err != nil {
410 return nil, nil, err
411 }
412
413 return c.NewBugRaw(author, time.Now().Unix(), title, message, files, nil)
414}
415
416// NewBugWithFilesMeta create a new bug with attached files for the message, as
417// well as metadata for the Create operation.
418// The new bug is written in the repository (commit)
419func (c *RepoCache) NewBugRaw(author *IdentityCache, unixTime int64, title string, message string, files []repository.Hash, metadata map[string]string) (*BugCache, *bug.CreateOperation, error) {
420 b, op, err := bug.CreateWithFiles(author.Identity, unixTime, title, message, files)
421 if err != nil {
422 return nil, nil, err
423 }
424
425 for key, value := range metadata {
426 op.SetMetadata(key, value)
427 }
428
429 err = b.Commit(c.repo)
430 if err != nil {
431 return nil, nil, err
432 }
433
434 c.muBug.Lock()
435 if _, has := c.bugs[b.Id()]; has {
436 c.muBug.Unlock()
437 return nil, nil, fmt.Errorf("bug %s already exist in the cache", b.Id())
438 }
439
440 cached := NewBugCache(c, b)
441 c.bugs[b.Id()] = cached
442 c.loadedBugs.Add(b.Id())
443 c.muBug.Unlock()
444
445 c.evictIfNeeded()
446
447 // force the write of the excerpt
448 err = c.bugUpdated(b.Id())
449 if err != nil {
450 return nil, nil, err
451 }
452
453 return cached, op, nil
454}
455
456// RemoveBug removes a bug from the cache and repo given a bug id prefix
457func (c *RepoCache) RemoveBug(prefix string) error {
458 c.muBug.RLock()
459
460 b, err := c.ResolveBugPrefix(prefix)
461 if err != nil {
462 c.muBug.RUnlock()
463 return err
464 }
465 c.muBug.RUnlock()
466
467 c.muBug.Lock()
468 err = bug.RemoveBug(c.repo, b.Id())
469
470 delete(c.bugs, b.Id())
471 delete(c.bugExcerpts, b.Id())
472 c.loadedBugs.Remove(b.Id())
473
474 c.muBug.Unlock()
475
476 return c.writeBugCache()
477}
478
479func (c *RepoCache) addBugToSearchIndex(snap *bug.Snapshot) error {
480 searchableBug := struct {
481 Text []string
482 }{}
483
484 // See https://github.com/blevesearch/bleve/issues/1576
485 var sb strings.Builder
486 normalize := func(text string) string {
487 sb.Reset()
488 for _, field := range strings.Fields(text) {
489 if utf8.RuneCountInString(field) < 100 {
490 sb.WriteString(field)
491 sb.WriteRune(' ')
492 }
493 }
494 return sb.String()
495 }
496
497 for _, comment := range snap.Comments {
498 searchableBug.Text = append(searchableBug.Text, normalize(comment.Message))
499 }
500
501 searchableBug.Text = append(searchableBug.Text, normalize(snap.Title))
502
503 index, err := c.repo.GetBleveIndex("bug")
504 if err != nil {
505 return err
506 }
507
508 err = index.Index(snap.Id().String(), searchableBug)
509 if err != nil {
510 return err
511 }
512
513 return nil
514}