1package cache
2
3import (
4 "bytes"
5 "encoding/gob"
6 "fmt"
7 "os"
8 "sync"
9
10 "github.com/pkg/errors"
11
12 "github.com/MichaelMure/git-bug/entities/bug"
13 "github.com/MichaelMure/git-bug/entity"
14 "github.com/MichaelMure/git-bug/repository"
15)
16
17type Excerpt interface {
18 Id() entity.Id
19}
20
21type CacheEntity interface {
22 NeedCommit() bool
23}
24
25type getUserIdentityFunc func() (*IdentityCache, error)
26
27type SubCache[ExcerptT Excerpt, CacheT CacheEntity, EntityT entity.Interface] struct {
28 repo repository.ClockedRepo
29 resolvers func() entity.Resolvers
30
31 getUserIdentity getUserIdentityFunc
32 readWithResolver func(repository.ClockedRepo, entity.Resolvers, entity.Id) (EntityT, error)
33 makeCached func(*SubCache[ExcerptT, CacheT, EntityT], getUserIdentityFunc, EntityT) CacheT
34 makeExcerpt func() Excerpt
35 indexingCallback func(CacheT) error
36
37 typename string
38 namespace string
39 version uint
40 maxLoaded int
41
42 mu sync.RWMutex
43 excerpts map[entity.Id]ExcerptT
44 cached map[entity.Id]CacheT
45 lru *lruIdCache
46}
47
48func NewSubCache[ExcerptT Excerpt, CacheT CacheEntity, EntityT entity.Interface](
49 repo repository.ClockedRepo,
50 resolvers func() entity.Resolvers,
51 getUserIdentity getUserIdentityFunc,
52 typename, namespace string,
53 version uint, maxLoaded int) *SubCache[ExcerptT, CacheT, EntityT] {
54 return &SubCache[ExcerptT, CacheT, EntityT]{
55 repo: repo,
56 resolvers: resolvers,
57 getUserIdentity: getUserIdentity,
58 typename: typename,
59 namespace: namespace,
60 version: version,
61 maxLoaded: maxLoaded,
62 excerpts: make(map[entity.Id]ExcerptT),
63 cached: make(map[entity.Id]CacheT),
64 lru: newLRUIdCache(),
65 }
66}
67
68func (sc *SubCache[ExcerptT, CacheT, EntityT]) Typename() string {
69 return sc.typename
70}
71
72// Load will try to read from the disk the entity cache file
73func (sc *SubCache[ExcerptT, CacheT, EntityT]) Load() error {
74 sc.mu.Lock()
75 defer sc.mu.Unlock()
76
77 f, err := sc.repo.LocalStorage().Open(sc.namespace + "-file")
78 if err != nil {
79 return err
80 }
81
82 decoder := gob.NewDecoder(f)
83
84 aux := struct {
85 Version uint
86 Excerpts map[entity.Id]ExcerptT
87 }{}
88
89 err = decoder.Decode(&aux)
90 if err != nil {
91 return err
92 }
93
94 if aux.Version != sc.version {
95 return fmt.Errorf("unknown %s cache format version %v", sc.namespace, aux.Version)
96 }
97
98 sc.excerpts = aux.Excerpts
99
100 index, err := sc.repo.GetBleveIndex("bug")
101 if err != nil {
102 return err
103 }
104
105 // simple heuristic to detect a mismatch between the index and the entities
106 count, err := index.DocCount()
107 if err != nil {
108 return err
109 }
110 if count != uint64(len(sc.excerpts)) {
111 return fmt.Errorf("count mismatch between bleve and %s excerpts", sc.namespace)
112 }
113
114 return nil
115}
116
117// Write will serialize on disk the entity cache file
118func (sc *SubCache[ExcerptT, CacheT, EntityT]) Write() error {
119 sc.mu.RLock()
120 defer sc.mu.RUnlock()
121
122 var data bytes.Buffer
123
124 aux := struct {
125 Version uint
126 Excerpts map[entity.Id]ExcerptT
127 }{
128 Version: sc.version,
129 Excerpts: sc.excerpts,
130 }
131
132 encoder := gob.NewEncoder(&data)
133
134 err := encoder.Encode(aux)
135 if err != nil {
136 return err
137 }
138
139 f, err := sc.repo.LocalStorage().Create(sc.namespace + "-file")
140 if err != nil {
141 return err
142 }
143
144 _, err = f.Write(data.Bytes())
145 if err != nil {
146 return err
147 }
148
149 return f.Close()
150}
151
152func (sc *SubCache[ExcerptT, CacheT, EntityT]) Build() error {
153 sc.excerpts = make(map[entity.Id]ExcerptT)
154
155 sc.readWithResolver
156
157 allBugs := bug.ReadAllWithResolver(c.repo, c.resolvers)
158
159 // wipe the index just to be sure
160 err := c.repo.ClearBleveIndex("bug")
161 if err != nil {
162 return err
163 }
164
165 for b := range allBugs {
166 if b.Err != nil {
167 return b.Err
168 }
169
170 snap := b.Bug.Compile()
171 c.bugExcerpts[b.Bug.Id()] = NewBugExcerpt(b.Bug, snap)
172
173 if err := c.addBugToSearchIndex(snap); err != nil {
174 return err
175 }
176 }
177
178 _, _ = fmt.Fprintln(os.Stderr, "Done.")
179}
180
181func (sc *SubCache[ExcerptT, CacheT, EntityT]) Close() error {
182 sc.mu.Lock()
183 defer sc.mu.Unlock()
184 sc.excerpts = nil
185 sc.cached = make(map[entity.Id]CacheT)
186 return nil
187}
188
189// AllIds return all known bug ids
190func (sc *SubCache[ExcerptT, CacheT, EntityT]) AllIds() []entity.Id {
191 sc.mu.RLock()
192 defer sc.mu.RUnlock()
193
194 result := make([]entity.Id, len(sc.excerpts))
195
196 i := 0
197 for _, excerpt := range sc.excerpts {
198 result[i] = excerpt.Id()
199 i++
200 }
201
202 return result
203}
204
205// Resolve retrieve an entity matching the exact given id
206func (sc *SubCache[ExcerptT, CacheT, EntityT]) Resolve(id entity.Id) (CacheT, error) {
207 sc.mu.RLock()
208 cached, ok := sc.cached[id]
209 if ok {
210 sc.lru.Get(id)
211 sc.mu.RUnlock()
212 return cached, nil
213 }
214 sc.mu.RUnlock()
215
216 b, err := sc.readWithResolver(sc.repo, sc.resolvers(), id)
217 if err != nil {
218 return *new(CacheT), err
219 }
220
221 cached = sc.makeCached(sc, sc.getUserIdentity, b)
222
223 sc.mu.Lock()
224 sc.cached[id] = cached
225 sc.lru.Add(id)
226 sc.mu.Unlock()
227
228 sc.evictIfNeeded()
229
230 return cached, nil
231}
232
233// ResolvePrefix retrieve an entity matching an id prefix. It fails if multiple
234// entity match.
235func (sc *SubCache[ExcerptT, CacheT, EntityT]) ResolvePrefix(prefix string) (CacheT, error) {
236 return sc.ResolveMatcher(func(excerpt ExcerptT) bool {
237 return excerpt.Id().HasPrefix(prefix)
238 })
239}
240
241func (sc *SubCache[ExcerptT, CacheT, EntityT]) ResolveMatcher(f func(ExcerptT) bool) (CacheT, error) {
242 id, err := sc.resolveMatcher(f)
243 if err != nil {
244 return *new(CacheT), err
245 }
246 return sc.Resolve(id)
247}
248
249// ResolveExcerpt retrieve an Excerpt matching the exact given id
250func (sc *SubCache[ExcerptT, CacheT, EntityT]) ResolveExcerpt(id entity.Id) (ExcerptT, error) {
251 sc.mu.RLock()
252 defer sc.mu.RUnlock()
253
254 excerpt, ok := sc.excerpts[id]
255 if !ok {
256 return *new(ExcerptT), entity.NewErrNotFound(sc.typename)
257 }
258
259 return excerpt, nil
260}
261
262// ResolveExcerptPrefix retrieve an Excerpt matching an id prefix. It fails if multiple
263// entity match.
264func (sc *SubCache[ExcerptT, CacheT, EntityT]) ResolveExcerptPrefix(prefix string) (ExcerptT, error) {
265 return sc.ResolveExcerptMatcher(func(excerpt ExcerptT) bool {
266 return excerpt.Id().HasPrefix(prefix)
267 })
268}
269
270func (sc *SubCache[ExcerptT, CacheT, EntityT]) ResolveExcerptMatcher(f func(ExcerptT) bool) (ExcerptT, error) {
271 id, err := sc.resolveMatcher(f)
272 if err != nil {
273 return *new(ExcerptT), err
274 }
275 return sc.ResolveExcerpt(id)
276}
277
278func (sc *SubCache[ExcerptT, CacheT, EntityT]) resolveMatcher(f func(ExcerptT) bool) (entity.Id, error) {
279 sc.mu.RLock()
280 defer sc.mu.RUnlock()
281
282 // preallocate but empty
283 matching := make([]entity.Id, 0, 5)
284
285 for _, excerpt := range sc.excerpts {
286 if f(excerpt) {
287 matching = append(matching, excerpt.Id())
288 }
289 }
290
291 if len(matching) > 1 {
292 return entity.UnsetId, entity.NewErrMultipleMatch(sc.typename, matching)
293 }
294
295 if len(matching) == 0 {
296 return entity.UnsetId, entity.NewErrNotFound(sc.typename)
297 }
298
299 return matching[0], nil
300}
301
302var errNotInCache = errors.New("entity missing from cache")
303
304func (sc *SubCache[ExcerptT, CacheT, EntityT]) add(e EntityT) (CacheT, error) {
305 sc.mu.Lock()
306 if _, has := sc.cached[e.Id()]; has {
307 sc.mu.Unlock()
308 return *new(CacheT), fmt.Errorf("entity %s already exist in the cache", e.Id())
309 }
310
311 cached := sc.makeCached(sc, sc.getUserIdentity, e)
312 sc.cached[e.Id()] = cached
313 sc.lru.Add(e.Id())
314 sc.mu.Unlock()
315
316 sc.evictIfNeeded()
317
318 // force the write of the excerpt
319 err := sc.entityUpdated(e.Id())
320 if err != nil {
321 return *new(CacheT), err
322 }
323
324 return cached, nil
325}
326
327func (sc *SubCache[ExcerptT, CacheT, EntityT]) Remove(prefix string) error {
328 e, err := sc.ResolvePrefix(prefix)
329 if err != nil {
330 return err
331 }
332
333 sc.mu.Lock()
334
335 err = bug.Remove(c.repo, b.Id())
336 if err != nil {
337 c.muBug.Unlock()
338
339 return err
340 }
341
342 delete(c.bugs, b.Id())
343 delete(c.bugExcerpts, b.Id())
344 c.loadedBugs.Remove(b.Id())
345
346 c.muBug.Unlock()
347
348 return c.writeBugCache()
349}
350
351// entityUpdated is a callback to trigger when the excerpt of an entity changed
352func (sc *SubCache[ExcerptT, CacheT, EntityT]) entityUpdated(id entity.Id) error {
353 sc.mu.Lock()
354 b, ok := sc.cached[id]
355 if !ok {
356 sc.mu.Unlock()
357
358 // if the bug is not loaded at this point, it means it was loaded before
359 // but got evicted. Which means we potentially have multiple copies in
360 // memory and thus concurrent write.
361 // Failing immediately here is the simple and safe solution to avoid
362 // complicated data loss.
363 return errNotInCache
364 }
365 sc.lru.Get(id)
366 // sc.excerpts[id] = bug2.NewBugExcerpt(b.bug, b.Snapshot())
367 sc.excerpts[id] = bug2.NewBugExcerpt(b.bug, b.Snapshot())
368 sc.mu.Unlock()
369
370 if err := sc.addBugToSearchIndex(b.Snapshot()); err != nil {
371 return err
372 }
373
374 // we only need to write the bug cache
375 return sc.Write()
376}
377
378// evictIfNeeded will evict an entity from the cache if needed
379func (sc *SubCache[ExcerptT, CacheT, EntityT]) evictIfNeeded() {
380 sc.mu.Lock()
381 defer sc.mu.Unlock()
382 if sc.lru.Len() <= sc.maxLoaded {
383 return
384 }
385
386 for _, id := range sc.lru.GetOldestToNewest() {
387 b := sc.cached[id]
388 if b.NeedCommit() {
389 continue
390 }
391
392 b.Lock()
393 sc.lru.Remove(id)
394 delete(sc.cached, id)
395
396 if sc.lru.Len() <= sc.maxLoaded {
397 return
398 }
399 }
400}