Detailed changes
@@ -30,7 +30,7 @@ var _ repository.RepoKeyring = &RepoCache{}
type cacheMgmt interface {
Typename() string
Load() error
- Build() error
+ Build() <-chan BuildEvent
SetCacheSize(size int)
RemoveAll() error
MergeAll(remote string) <-chan entity.MergeResult
@@ -213,6 +213,7 @@ const (
BuildEventCacheIsBuilt
BuildEventRemoveLock
BuildEventStarted
+ BuildEventProgress
BuildEventFinished
)
@@ -224,6 +225,10 @@ type BuildEvent struct {
Typename string
// Event is the type of the event.
Event BuildEventType
+ // Total is the total number of element being built. Set if Event is BuildEventStarted.
+ Total int64
+ // Progress is the current count of processed element. Set if Event is BuildEventProgress.
+ Progress int64
}
func (c *RepoCache) buildCache(events chan BuildEvent) {
@@ -234,23 +239,13 @@ func (c *RepoCache) buildCache(events chan BuildEvent) {
wg.Add(1)
go func(subcache cacheMgmt) {
defer wg.Done()
- events <- BuildEvent{
- Typename: subcache.Typename(),
- Event: BuildEventStarted,
- }
- err := subcache.Build()
- if err != nil {
- events <- BuildEvent{
- Typename: subcache.Typename(),
- Err: err,
+ buildEvents := subcache.Build()
+ for buildEvent := range buildEvents {
+ events <- buildEvent
+ if buildEvent.Err != nil {
+ return
}
- return
- }
-
- events <- BuildEvent{
- Typename: subcache.Typename(),
- Event: BuildEventFinished,
}
}(subcache)
}
@@ -186,51 +186,98 @@ func (sc *SubCache[EntityT, ExcerptT, CacheT]) write() error {
return f.Close()
}
-func (sc *SubCache[EntityT, ExcerptT, CacheT]) Build() error {
- sc.excerpts = make(map[entity.Id]ExcerptT)
+func (sc *SubCache[EntityT, ExcerptT, CacheT]) Build() <-chan BuildEvent {
+ out := make(chan BuildEvent)
- allEntities := sc.actions.ReadAllWithResolver(sc.repo, sc.resolvers())
+ go func() {
+ defer close(out)
- index, err := sc.repo.GetIndex(sc.namespace)
- if err != nil {
- return err
- }
+ out <- BuildEvent{
+ Typename: sc.typename,
+ Event: BuildEventStarted,
+ }
- // wipe the index just to be sure
- err = index.Clear()
- if err != nil {
- return err
- }
+ sc.excerpts = make(map[entity.Id]ExcerptT)
- indexer, indexEnd := index.IndexBatch()
+ allEntities := sc.actions.ReadAllWithResolver(sc.repo, sc.resolvers())
- for e := range allEntities {
- if e.Err != nil {
- return e.Err
+ index, err := sc.repo.GetIndex(sc.namespace)
+ if err != nil {
+ out <- BuildEvent{
+ Typename: sc.typename,
+ Err: err,
+ }
+ return
}
- cached := sc.makeCached(e.Entity, sc.entityUpdated)
- sc.excerpts[e.Entity.Id()] = sc.makeExcerpt(cached)
- // might as well keep them in memory
- sc.cached[e.Entity.Id()] = cached
+ // wipe the index just to be sure
+ err = index.Clear()
+ if err != nil {
+ out <- BuildEvent{
+ Typename: sc.typename,
+ Err: err,
+ }
+ return
+ }
+
+ indexer, indexEnd := index.IndexBatch()
+
+ for e := range allEntities {
+ if e.Err != nil {
+ out <- BuildEvent{
+ Typename: sc.typename,
+ Err: e.Err,
+ }
+ return
+ }
+
+ cached := sc.makeCached(e.Entity, sc.entityUpdated)
+ sc.excerpts[e.Entity.Id()] = sc.makeExcerpt(cached)
+ // might as well keep them in memory
+ sc.cached[e.Entity.Id()] = cached
+
+ indexData := sc.makeIndexData(cached)
+ if err := indexer(e.Entity.Id().String(), indexData); err != nil {
+ out <- BuildEvent{
+ Typename: sc.typename,
+ Err: err,
+ }
+ return
+ }
- indexData := sc.makeIndexData(cached)
- if err := indexer(e.Entity.Id().String(), indexData); err != nil {
- return err
+ out <- BuildEvent{
+ Typename: sc.typename,
+ Event: BuildEventProgress,
+ Progress: e.CurrentEntity,
+ Total: e.TotalEntities,
+ }
}
- }
- err = indexEnd()
- if err != nil {
- return err
- }
+ err = indexEnd()
+ if err != nil {
+ out <- BuildEvent{
+ Typename: sc.typename,
+ Err: err,
+ }
+ return
+ }
- err = sc.write()
- if err != nil {
- return err
- }
+ err = sc.write()
+ if err != nil {
+ out <- BuildEvent{
+ Typename: sc.typename,
+ Err: err,
+ }
+ return
+ }
- return nil
+ out <- BuildEvent{
+ Typename: sc.typename,
+ Event: BuildEventFinished,
+ }
+ }()
+
+ return out
}
func (sc *SubCache[EntityT, ExcerptT, CacheT]) SetCacheSize(size int) {
@@ -7,6 +7,8 @@ import (
"os"
"github.com/spf13/cobra"
+ "github.com/vbauerster/mpb/v8"
+ "github.com/vbauerster/mpb/v8/decor"
"github.com/MichaelMure/git-bug/cache"
"github.com/MichaelMure/git-bug/entities/identity"
@@ -50,6 +52,10 @@ type Out interface {
// Reset clear what has been recorded as written in the output before.
// This only works in test scenario.
Reset()
+
+ // Raw return the underlying io.Writer, or itself if not.
+ // This is useful if something need to access the raw file descriptor.
+ Raw() io.Writer
}
type out struct {
@@ -89,6 +95,10 @@ func (o out) Reset() {
panic("only work with a test env")
}
+func (o out) Raw() io.Writer {
+ return o.Writer
+}
+
// LoadRepo is a pre-run function that load the repository for use in a command
func LoadRepo(env *Env) func(*cobra.Command, []string) error {
return func(cmd *cobra.Command, args []string) error {
@@ -143,18 +153,9 @@ func LoadBackend(env *Env) func(*cobra.Command, []string) error {
var events chan cache.BuildEvent
env.Backend, events = cache.NewRepoCache(env.Repo)
- for event := range events {
- if event.Err != nil {
- return event.Err
- }
- switch event.Event {
- case cache.BuildEventCacheIsBuilt:
- env.Err.Println("Building cache... ")
- case cache.BuildEventStarted:
- env.Err.Printf("[%s] started\n", event.Typename)
- case cache.BuildEventFinished:
- env.Err.Printf("[%s] done\n", event.Typename)
- }
+ err = CacheBuildProgressBar(env, events)
+ if err != nil {
+ return err
}
cleaner := func(env *Env) interrupt.CleanerFunc {
@@ -213,3 +214,41 @@ func CloseBackend(env *Env, runE func(cmd *cobra.Command, args []string) error)
return err
}
}
+
+func CacheBuildProgressBar(env *Env, events chan cache.BuildEvent) error {
+ var progress *mpb.Progress
+ var bars = make(map[string]*mpb.Bar)
+
+ for event := range events {
+ if event.Err != nil {
+ return event.Err
+ }
+
+ if progress == nil {
+ progress = mpb.New(mpb.WithOutput(env.Err.Raw()))
+ }
+
+ switch event.Event {
+ case cache.BuildEventCacheIsBuilt:
+ env.Err.Println("Building cache... ")
+ case cache.BuildEventStarted:
+ bars[event.Typename] = progress.AddBar(-1,
+ mpb.BarRemoveOnComplete(),
+ mpb.PrependDecorators(
+ decor.Name(event.Typename, decor.WCSyncSpace),
+ decor.CountersNoUnit("%d / %d", decor.WCSyncSpace),
+ ),
+ mpb.AppendDecorators(decor.Percentage(decor.WCSyncSpace)),
+ )
+ case cache.BuildEventProgress:
+ bars[event.Typename].SetTotal(event.Total, false)
+ bars[event.Typename].SetCurrent(event.Progress)
+ }
+ }
+
+ if progress != nil {
+ progress.Shutdown()
+ }
+
+ return nil
+}
@@ -4,6 +4,7 @@ import (
"bytes"
"encoding/json"
"fmt"
+ "io"
"testing"
"github.com/stretchr/testify/require"
@@ -12,6 +13,8 @@ import (
"github.com/MichaelMure/git-bug/repository"
)
+var _ Out = &TestOut{}
+
type TestOut struct {
*bytes.Buffer
}
@@ -37,6 +40,10 @@ func (te *TestOut) PrintJSON(v interface{}) error {
return nil
}
+func (te *TestOut) Raw() io.Writer {
+ return te.Buffer
+}
+
func NewTestEnv(t *testing.T) *Env {
t.Helper()
@@ -108,19 +108,10 @@ func runWebUI(env *execenv.Env, opts webUIOptions) error {
mrc := cache.NewMultiRepoCache()
_, events := mrc.RegisterDefaultRepository(env.Repo)
- for event := range events {
- if event.Err != nil {
- env.Err.Printf("Cache building error [%s]: %v\n", event.Typename, event.Err)
- continue
- }
- switch event.Event {
- case cache.BuildEventCacheIsBuilt:
- env.Err.Println("Building cache... ")
- case cache.BuildEventStarted:
- env.Err.Printf("[%s] started\n", event.Typename)
- case cache.BuildEventFinished:
- env.Err.Printf("[%s] done\n", event.Typename)
- }
+
+ err := execenv.CacheBuildProgressBar(env, events)
+ if err != nil {
+ return err
}
var errOut io.Writer
@@ -188,6 +188,9 @@ func readAll(repo repository.ClockedRepo, refPrefix string) <-chan entity.Stream
return
}
+ total := int64(len(refs))
+ current := int64(1)
+
for _, ref := range refs {
i, err := read(repo, ref)
@@ -196,7 +199,12 @@ func readAll(repo repository.ClockedRepo, refPrefix string) <-chan entity.Stream
return
}
- out <- entity.StreamedEntity[*Identity]{Entity: i}
+ out <- entity.StreamedEntity[*Identity]{
+ Entity: i,
+ CurrentEntity: current,
+ TotalEntities: total,
+ }
+ current++
}
}()
@@ -314,6 +314,9 @@ func ReadAll[EntityT entity.Interface](def Definition, wrapper func(e *Entity) E
return
}
+ total := int64(len(refs))
+ current := int64(1)
+
for _, ref := range refs {
e, err := read[EntityT](def, wrapper, repo, resolvers, ref)
@@ -322,7 +325,12 @@ func ReadAll[EntityT entity.Interface](def Definition, wrapper func(e *Entity) E
return
}
- out <- entity.StreamedEntity[EntityT]{Entity: e}
+ out <- entity.StreamedEntity[EntityT]{
+ Entity: e,
+ CurrentEntity: current,
+ TotalEntities: total,
+ }
+ current++
}
}()
@@ -1,6 +1,11 @@
package entity
type StreamedEntity[EntityT Interface] struct {
- Entity EntityT
Err error
+ Entity EntityT
+
+ // CurrentEntity is the index of the current entity being streamed, to express progress.
+ CurrentEntity int64
+ // TotalEntities is the total count of expected entities, if known.
+ TotalEntities int64
}
@@ -26,6 +26,7 @@ require (
github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e
github.com/spf13/cobra v1.6.1
github.com/stretchr/testify v1.8.1
+ github.com/vbauerster/mpb/v8 v8.1.4
github.com/vektah/gqlparser/v2 v2.5.1
github.com/xanzy/go-gitlab v0.77.0
golang.org/x/crypto v0.5.0
@@ -39,6 +40,8 @@ require (
replace github.com/go-git/go-git/v5 => github.com/MichaelMure/go-git/v5 v5.1.1-0.20230114115943-17400561a81c
require (
+ github.com/VividCortex/ewma v1.2.0 // indirect
+ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/cloudflare/circl v1.3.1 // indirect
github.com/lithammer/dedent v1.1.0 // indirect
github.com/owenrumney/go-sarif v1.0.11 // indirect
@@ -89,14 +92,14 @@ require (
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
- github.com/mattn/go-runewidth v0.0.12 // indirect
+ github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/mtibben/percent v0.2.1 // indirect
github.com/philhofer/fwd v1.0.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/praetorian-inc/gokart v0.5.1
- github.com/rivo/uniseg v0.1.0 // indirect
+ github.com/rivo/uniseg v0.2.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sergi/go-diff v1.2.0 // indirect
github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f // indirect
@@ -21,6 +21,10 @@ github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0A
github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
github.com/RoaringBitmap/roaring v0.4.23 h1:gpyfd12QohbqhFO4NVDUdoPOCXsyahYRQhINmlHxKeo=
github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo=
+github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
+github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
@@ -209,8 +213,9 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
-github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow=
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
+github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@@ -243,8 +248,9 @@ github.com/praetorian-inc/gokart v0.5.1 h1:GYUM69qskrRibZUAEwKEm/pd/j/SFzlFnQnhx
github.com/praetorian-inc/gokart v0.5.1/go.mod h1:GuA97YgdXwqOVsnHY6PCvV1t9t0Jsk3Zcd6sbTXj4uI=
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -298,6 +304,8 @@ github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDW
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/urfave/cli/v2 v2.8.1 h1:CGuYNZF9IKZY/rfBe3lJpccSoIY1ytfvmgQT90cNOl4=
github.com/urfave/cli/v2 v2.8.1/go.mod h1:Z41J9TPoffeoqP0Iza0YbAhGvymRdZAd2uPmZ5JxRdY=
+github.com/vbauerster/mpb/v8 v8.1.4 h1:MOcLTIbbAA892wVjRiuFHa1nRlNvifQMDVh12Bq/xIs=
+github.com/vbauerster/mpb/v8 v8.1.4/go.mod h1:2fRME8lCLU9gwJwghZb1bO9A3Plc8KPeQ/ayGj+Ek4I=
github.com/vektah/gqlparser/v2 v2.5.1 h1:ZGu+bquAY23jsxDRcYpWjttRZrUz07LbiY77gUOHcr4=
github.com/vektah/gqlparser/v2 v2.5.1/go.mod h1:mPgqFBu/woKTVYWyNk8cO3kh4S/f4aRFZrvOnp3hmCs=
github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=