1package cache
2
3import (
4 "bytes"
5 "encoding/gob"
6 "errors"
7 "fmt"
8 "sort"
9 "strings"
10 "time"
11 "unicode/utf8"
12
13 "github.com/blevesearch/bleve"
14
15 "github.com/MichaelMure/git-bug/entities/bug"
16 "github.com/MichaelMure/git-bug/entity"
17 "github.com/MichaelMure/git-bug/query"
18 "github.com/MichaelMure/git-bug/repository"
19)
20
21const bugCacheFile = "bug-cache"
22
23var errBugNotInCache = errors.New("bug missing from cache")
24
25// bugUpdated is a callback to trigger when the excerpt of a bug changed,
26// that is each time a bug is updated
27func (c *RepoCache) bugUpdated(id entity.Id) error {
28 c.muBug.Lock()
29 b, ok := c.bugs[id]
30 if !ok {
31 c.muBug.Unlock()
32
33 // if the bug is not loaded at this point, it means it was loaded before
34 // but got evicted. Which means we potentially have multiple copies in
35 // memory and thus concurrent write.
36 // Failing immediately here is the simple and safe solution to avoid
37 // complicated data loss.
38 return errBugNotInCache
39 }
40 c.loadedBugs.Get(id)
41 c.bugExcerpts[id] = NewBugExcerpt(b.bug, b.Snapshot())
42 c.muBug.Unlock()
43
44 if err := c.addBugToSearchIndex(b.Snapshot()); err != nil {
45 return err
46 }
47
48 // we only need to write the bug cache
49 return c.writeBugCache()
50}
51
52// load will try to read from the disk the bug cache file
53func (c *RepoCache) loadBugCache() error {
54 c.muBug.Lock()
55 defer c.muBug.Unlock()
56
57 f, err := c.repo.LocalStorage().Open(bugCacheFile)
58 if err != nil {
59 return err
60 }
61
62 decoder := gob.NewDecoder(f)
63
64 aux := struct {
65 Version uint
66 Excerpts map[entity.Id]*BugExcerpt
67 }{}
68
69 err = decoder.Decode(&aux)
70 if err != nil {
71 return err
72 }
73
74 if aux.Version != formatVersion {
75 return fmt.Errorf("unknown cache format version %v", aux.Version)
76 }
77
78 c.bugExcerpts = aux.Excerpts
79
80 index, err := c.repo.GetBleveIndex("bug")
81 if err != nil {
82 return err
83 }
84
85 // simple heuristic to detect a mismatch between the index and the bugs
86 count, err := index.DocCount()
87 if err != nil {
88 return err
89 }
90 if count != uint64(len(c.bugExcerpts)) {
91 return fmt.Errorf("count mismatch between bleve and bug excerpts")
92 }
93
94 return nil
95}
96
97// write will serialize on disk the bug cache file
98func (c *RepoCache) writeBugCache() error {
99 c.muBug.RLock()
100 defer c.muBug.RUnlock()
101
102 var data bytes.Buffer
103
104 aux := struct {
105 Version uint
106 Excerpts map[entity.Id]*BugExcerpt
107 }{
108 Version: formatVersion,
109 Excerpts: c.bugExcerpts,
110 }
111
112 encoder := gob.NewEncoder(&data)
113
114 err := encoder.Encode(aux)
115 if err != nil {
116 return err
117 }
118
119 f, err := c.repo.LocalStorage().Create(bugCacheFile)
120 if err != nil {
121 return err
122 }
123
124 _, err = f.Write(data.Bytes())
125 if err != nil {
126 return err
127 }
128
129 return f.Close()
130}
131
132// ResolveBugExcerpt retrieve a BugExcerpt matching the exact given id
133func (c *RepoCache) ResolveBugExcerpt(id entity.Id) (*BugExcerpt, error) {
134 c.muBug.RLock()
135 defer c.muBug.RUnlock()
136
137 excerpt, ok := c.bugExcerpts[id]
138 if !ok {
139 return nil, bug.ErrBugNotExist
140 }
141
142 return excerpt, nil
143}
144
145// ResolveBug retrieve a bug matching the exact given id
146func (c *RepoCache) ResolveBug(id entity.Id) (*BugCache, error) {
147 c.muBug.RLock()
148 cached, ok := c.bugs[id]
149 if ok {
150 c.loadedBugs.Get(id)
151 c.muBug.RUnlock()
152 return cached, nil
153 }
154 c.muBug.RUnlock()
155
156 b, err := bug.ReadWithResolver(c.repo, c.resolvers, id)
157 if err != nil {
158 return nil, err
159 }
160
161 cached = NewBugCache(c, b)
162
163 c.muBug.Lock()
164 c.bugs[id] = cached
165 c.loadedBugs.Add(id)
166 c.muBug.Unlock()
167
168 c.evictIfNeeded()
169
170 return cached, nil
171}
172
173// evictIfNeeded will evict a bug from the cache if needed
174// it also removes references of the bug from the bugs
175func (c *RepoCache) evictIfNeeded() {
176 c.muBug.Lock()
177 defer c.muBug.Unlock()
178 if c.loadedBugs.Len() <= c.maxLoadedBugs {
179 return
180 }
181
182 for _, id := range c.loadedBugs.GetOldestToNewest() {
183 b := c.bugs[id]
184 if b.NeedCommit() {
185 continue
186 }
187
188 b.mu.Lock()
189 c.loadedBugs.Remove(id)
190 delete(c.bugs, id)
191
192 if c.loadedBugs.Len() <= c.maxLoadedBugs {
193 return
194 }
195 }
196}
197
198// ResolveBugExcerptPrefix retrieve a BugExcerpt matching an id prefix. It fails if multiple
199// bugs match.
200func (c *RepoCache) ResolveBugExcerptPrefix(prefix string) (*BugExcerpt, error) {
201 return c.ResolveBugExcerptMatcher(func(excerpt *BugExcerpt) bool {
202 return excerpt.Id.HasPrefix(prefix)
203 })
204}
205
206// ResolveBugPrefix retrieve a bug matching an id prefix. It fails if multiple
207// bugs match.
208func (c *RepoCache) ResolveBugPrefix(prefix string) (*BugCache, error) {
209 return c.ResolveBugMatcher(func(excerpt *BugExcerpt) bool {
210 return excerpt.Id.HasPrefix(prefix)
211 })
212}
213
214// ResolveBugCreateMetadata retrieve a bug that has the exact given metadata on
215// its Create operation, that is, the first operation. It fails if multiple bugs
216// match.
217func (c *RepoCache) ResolveBugCreateMetadata(key string, value string) (*BugCache, error) {
218 return c.ResolveBugMatcher(func(excerpt *BugExcerpt) bool {
219 return excerpt.CreateMetadata[key] == value
220 })
221}
222
223func (c *RepoCache) ResolveBugExcerptMatcher(f func(*BugExcerpt) bool) (*BugExcerpt, error) {
224 id, err := c.resolveBugMatcher(f)
225 if err != nil {
226 return nil, err
227 }
228 return c.ResolveBugExcerpt(id)
229}
230
231func (c *RepoCache) ResolveBugMatcher(f func(*BugExcerpt) bool) (*BugCache, error) {
232 id, err := c.resolveBugMatcher(f)
233 if err != nil {
234 return nil, err
235 }
236 return c.ResolveBug(id)
237}
238
239func (c *RepoCache) resolveBugMatcher(f func(*BugExcerpt) bool) (entity.Id, error) {
240 c.muBug.RLock()
241 defer c.muBug.RUnlock()
242
243 // preallocate but empty
244 matching := make([]entity.Id, 0, 5)
245
246 for _, excerpt := range c.bugExcerpts {
247 if f(excerpt) {
248 matching = append(matching, excerpt.Id)
249 }
250 }
251
252 if len(matching) > 1 {
253 return entity.UnsetId, bug.NewErrMultipleMatchBug(matching)
254 }
255
256 if len(matching) == 0 {
257 return entity.UnsetId, bug.ErrBugNotExist
258 }
259
260 return matching[0], nil
261}
262
263// ResolveComment search for a Bug/Comment combination matching the merged
264// bug/comment Id prefix. Returns the Bug containing the Comment and the Comment's
265// Id.
266func (c *RepoCache) ResolveComment(prefix string) (*BugCache, entity.CombinedId, error) {
267 bugPrefix, _ := entity.SeparateIds(prefix)
268 bugCandidate := make([]entity.Id, 0, 5)
269
270 // build a list of possible matching bugs
271 c.muBug.RLock()
272 for _, excerpt := range c.bugExcerpts {
273 if excerpt.Id.HasPrefix(bugPrefix) {
274 bugCandidate = append(bugCandidate, excerpt.Id)
275 }
276 }
277 c.muBug.RUnlock()
278
279 matchingBugIds := make([]entity.Id, 0, 5)
280 matchingCommentId := entity.UnsetCombinedId
281 var matchingBug *BugCache
282
283 // search for matching comments
284 // searching every bug candidate allow for some collision with the bug prefix only,
285 // before being refined with the full comment prefix
286 for _, bugId := range bugCandidate {
287 b, err := c.ResolveBug(bugId)
288 if err != nil {
289 return nil, entity.UnsetCombinedId, err
290 }
291
292 for _, comment := range b.Snapshot().Comments {
293 if comment.TargetId().HasPrefix(prefix) {
294 matchingBugIds = append(matchingBugIds, bugId)
295 matchingBug = b
296 matchingCommentId = comment.CombinedId()
297 }
298 }
299 }
300
301 if len(matchingBugIds) > 1 {
302 return nil, entity.UnsetCombinedId, entity.NewErrMultipleMatch("bug/comment", matchingBugIds)
303 } else if len(matchingBugIds) == 0 {
304 return nil, entity.UnsetCombinedId, errors.New("comment doesn't exist")
305 }
306
307 return matchingBug, matchingCommentId, nil
308}
309
310// QueryBugs return the id of all Bug matching the given Query
311func (c *RepoCache) QueryBugs(q *query.Query) ([]entity.Id, error) {
312 c.muBug.RLock()
313 defer c.muBug.RUnlock()
314
315 if q == nil {
316 return c.AllBugsIds(), nil
317 }
318
319 matcher := compileMatcher(q.Filters)
320
321 var filtered []*BugExcerpt
322 var foundBySearch map[entity.Id]*BugExcerpt
323
324 if q.Search != nil {
325 foundBySearch = map[entity.Id]*BugExcerpt{}
326
327 terms := make([]string, len(q.Search))
328 copy(terms, q.Search)
329 for i, search := range q.Search {
330 if strings.Contains(search, " ") {
331 terms[i] = fmt.Sprintf("\"%s\"", search)
332 }
333 }
334
335 bleveQuery := bleve.NewQueryStringQuery(strings.Join(terms, " "))
336 bleveSearch := bleve.NewSearchRequest(bleveQuery)
337
338 index, err := c.repo.GetBleveIndex("bug")
339 if err != nil {
340 return nil, err
341 }
342
343 searchResults, err := index.Search(bleveSearch)
344 if err != nil {
345 return nil, err
346 }
347
348 for _, hit := range searchResults.Hits {
349 foundBySearch[entity.Id(hit.ID)] = c.bugExcerpts[entity.Id(hit.ID)]
350 }
351 } else {
352 foundBySearch = c.bugExcerpts
353 }
354
355 for _, excerpt := range foundBySearch {
356 if matcher.Match(excerpt, c) {
357 filtered = append(filtered, excerpt)
358 }
359 }
360
361 var sorter sort.Interface
362
363 switch q.OrderBy {
364 case query.OrderById:
365 sorter = BugsById(filtered)
366 case query.OrderByCreation:
367 sorter = BugsByCreationTime(filtered)
368 case query.OrderByEdit:
369 sorter = BugsByEditTime(filtered)
370 default:
371 return nil, errors.New("missing sort type")
372 }
373
374 switch q.OrderDirection {
375 case query.OrderAscending:
376 // Nothing to do
377 case query.OrderDescending:
378 sorter = sort.Reverse(sorter)
379 default:
380 return nil, errors.New("missing sort direction")
381 }
382
383 sort.Sort(sorter)
384
385 result := make([]entity.Id, len(filtered))
386
387 for i, val := range filtered {
388 result[i] = val.Id
389 }
390
391 return result, nil
392}
393
394// AllBugsIds return all known bug ids
395func (c *RepoCache) AllBugsIds() []entity.Id {
396 c.muBug.RLock()
397 defer c.muBug.RUnlock()
398
399 result := make([]entity.Id, len(c.bugExcerpts))
400
401 i := 0
402 for _, excerpt := range c.bugExcerpts {
403 result[i] = excerpt.Id
404 i++
405 }
406
407 return result
408}
409
410// ValidLabels list valid labels
411//
412// Note: in the future, a proper label policy could be implemented where valid
413// labels are defined in a configuration file. Until that, the default behavior
414// is to return the list of labels already used.
415func (c *RepoCache) ValidLabels() []bug.Label {
416 c.muBug.RLock()
417 defer c.muBug.RUnlock()
418
419 set := map[bug.Label]interface{}{}
420
421 for _, excerpt := range c.bugExcerpts {
422 for _, l := range excerpt.Labels {
423 set[l] = nil
424 }
425 }
426
427 result := make([]bug.Label, len(set))
428
429 i := 0
430 for l := range set {
431 result[i] = l
432 i++
433 }
434
435 // Sort
436 sort.Slice(result, func(i, j int) bool {
437 return string(result[i]) < string(result[j])
438 })
439
440 return result
441}
442
443// NewBug create a new bug
444// The new bug is written in the repository (commit)
445func (c *RepoCache) NewBug(title string, message string) (*BugCache, *bug.CreateOperation, error) {
446 return c.NewBugWithFiles(title, message, nil)
447}
448
449// NewBugWithFiles create a new bug with attached files for the message
450// The new bug is written in the repository (commit)
451func (c *RepoCache) NewBugWithFiles(title string, message string, files []repository.Hash) (*BugCache, *bug.CreateOperation, error) {
452 author, err := c.GetUserIdentity()
453 if err != nil {
454 return nil, nil, err
455 }
456
457 return c.NewBugRaw(author, time.Now().Unix(), title, message, files, nil)
458}
459
460// NewBugRaw create a new bug with attached files for the message, as
461// well as metadata for the Create operation.
462// The new bug is written in the repository (commit)
463func (c *RepoCache) NewBugRaw(author *IdentityCache, unixTime int64, title string, message string, files []repository.Hash, metadata map[string]string) (*BugCache, *bug.CreateOperation, error) {
464 b, op, err := bug.Create(author.Identity, unixTime, title, message, files, metadata)
465 if err != nil {
466 return nil, nil, err
467 }
468
469 err = b.Commit(c.repo)
470 if err != nil {
471 return nil, nil, err
472 }
473
474 c.muBug.Lock()
475 if _, has := c.bugs[b.Id()]; has {
476 c.muBug.Unlock()
477 return nil, nil, fmt.Errorf("bug %s already exist in the cache", b.Id())
478 }
479
480 cached := NewBugCache(c, b)
481 c.bugs[b.Id()] = cached
482 c.loadedBugs.Add(b.Id())
483 c.muBug.Unlock()
484
485 c.evictIfNeeded()
486
487 // force the write of the excerpt
488 err = c.bugUpdated(b.Id())
489 if err != nil {
490 return nil, nil, err
491 }
492
493 return cached, op, nil
494}
495
496// RemoveBug removes a bug from the cache and repo given a bug id prefix
497func (c *RepoCache) RemoveBug(prefix string) error {
498 b, err := c.ResolveBugPrefix(prefix)
499 if err != nil {
500 return err
501 }
502
503 c.muBug.Lock()
504 err = bug.RemoveBug(c.repo, b.Id())
505
506 delete(c.bugs, b.Id())
507 delete(c.bugExcerpts, b.Id())
508 c.loadedBugs.Remove(b.Id())
509
510 c.muBug.Unlock()
511
512 return c.writeBugCache()
513}
514
515func (c *RepoCache) addBugToSearchIndex(snap *bug.Snapshot) error {
516 searchableBug := struct {
517 Text []string
518 }{}
519
520 // See https://github.com/blevesearch/bleve/issues/1576
521 var sb strings.Builder
522 normalize := func(text string) string {
523 sb.Reset()
524 for _, field := range strings.Fields(text) {
525 if utf8.RuneCountInString(field) < 100 {
526 sb.WriteString(field)
527 sb.WriteRune(' ')
528 }
529 }
530 return sb.String()
531 }
532
533 for _, comment := range snap.Comments {
534 searchableBug.Text = append(searchableBug.Text, normalize(comment.Message))
535 }
536
537 searchableBug.Text = append(searchableBug.Text, normalize(snap.Title))
538
539 index, err := c.repo.GetBleveIndex("bug")
540 if err != nil {
541 return err
542 }
543
544 err = index.Index(snap.Id().String(), searchableBug)
545 if err != nil {
546 return err
547 }
548
549 return nil
550}