1package github
2
3import (
4 "bytes"
5 "context"
6 "encoding/json"
7 "fmt"
8 "io/ioutil"
9 "net/http"
10 "os"
11 "strings"
12 "time"
13
14 "github.com/pkg/errors"
15 "github.com/shurcooL/githubv4"
16 "golang.org/x/sync/errgroup"
17
18 "github.com/MichaelMure/git-bug/bridge/core"
19 "github.com/MichaelMure/git-bug/bridge/core/auth"
20 "github.com/MichaelMure/git-bug/bug"
21 "github.com/MichaelMure/git-bug/cache"
22 "github.com/MichaelMure/git-bug/entity"
23 "github.com/MichaelMure/git-bug/identity"
24 "github.com/MichaelMure/git-bug/repository"
25)
26
27var (
28 ErrMissingIdentityToken = errors.New("missing identity token")
29)
30
31// githubExporter implement the Exporter interface
32type githubExporter struct {
33 conf core.Configuration
34
35 // cache identities clients
36 identityClient map[entity.Id]*githubv4.Client
37
38 // github repository ID
39 repositoryID string
40
41 // cache identifiers used to speed up exporting operations
42 // cleared for each bug
43 cachedOperationIDs map[entity.Id]string
44
45 // cache labels used to speed up exporting labels events
46 cachedLabels map[string]string
47}
48
49// Init .
50func (ge *githubExporter) Init(repo *cache.RepoCache, conf core.Configuration) error {
51 ge.conf = conf
52 ge.identityClient = make(map[entity.Id]*githubv4.Client)
53 ge.cachedOperationIDs = make(map[entity.Id]string)
54 ge.cachedLabels = make(map[string]string)
55
56 // preload all clients
57 err := ge.cacheAllClient(repo)
58 if err != nil {
59 return err
60 }
61
62 return nil
63}
64
65func (ge *githubExporter) cacheAllClient(repo *cache.RepoCache) error {
66 creds, err := auth.List(repo, auth.WithTarget(target), auth.WithKind(auth.KindToken))
67 if err != nil {
68 return err
69 }
70
71 for _, cred := range creds {
72 login, ok := cred.Metadata()[auth.MetaKeyLogin]
73 if !ok {
74 _, _ = fmt.Fprintf(os.Stderr, "credential %s is not tagged with Github login\n", cred.ID().Human())
75 continue
76 }
77
78 user, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, login)
79 if err == identity.ErrIdentityNotExist {
80 continue
81 }
82
83 if _, ok := ge.identityClient[cred.UserId()]; !ok {
84 client := buildClient(creds[0].(*auth.Token))
85 ge.identityClient[cred.UserId()] = client
86 }
87 }
88
89 return nil
90}
91
92// getClientForIdentity return a githubv4 API client configured with the access token of the given identity.
93func (ge *githubExporter) getClientForIdentity(userId entity.Id) (*githubv4.Client, error) {
94 client, ok := ge.identityClient[userId]
95 if ok {
96 return client, nil
97 }
98
99 return nil, ErrMissingIdentityToken
100}
101
102// ExportAll export all event made by the current user to Github
103func (ge *githubExporter) ExportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ExportResult, error) {
104 out := make(chan core.ExportResult)
105
106 var err error
107 // get repository node id
108 ge.repositoryID, err = getRepositoryNodeID(
109 ctx,
110 ge.defaultToken,
111 ge.conf[keyOwner],
112 ge.conf[keyProject],
113 )
114 if err != nil {
115 return nil, err
116 }
117
118 // query all labels
119 err = ge.cacheGithubLabels(ctx, ge.defaultClient)
120 if err != nil {
121 return nil, err
122 }
123
124 go func() {
125 defer close(out)
126
127 allIdentitiesIds := make([]entity.Id, 0, len(ge.identityClient))
128 for id := range ge.identityClient {
129 allIdentitiesIds = append(allIdentitiesIds, id)
130 }
131
132 allBugsIds := repo.AllBugsIds()
133
134 for _, id := range allBugsIds {
135 b, err := repo.ResolveBug(id)
136 if err != nil {
137 out <- core.NewExportError(errors.Wrap(err, "can't load bug"), id)
138 return
139 }
140
141 select {
142
143 case <-ctx.Done():
144 // stop iterating if context cancel function is called
145 return
146
147 default:
148 snapshot := b.Snapshot()
149
150 // ignore issues created before since date
151 // TODO: compare the Lamport time instead of using the unix time
152 if snapshot.CreatedAt.Before(since) {
153 out <- core.NewExportNothing(b.Id(), "bug created before the since date")
154 continue
155 }
156
157 if snapshot.HasAnyActor(allIdentitiesIds...) {
158 // try to export the bug and it associated events
159 ge.exportBug(ctx, b, since, out)
160 }
161 }
162 }
163 }()
164
165 return out, nil
166}
167
168// exportBug publish bugs and related events
169func (ge *githubExporter) exportBug(ctx context.Context, b *cache.BugCache, since time.Time, out chan<- core.ExportResult) {
170 snapshot := b.Snapshot()
171 var bugUpdated bool
172
173 var bugGithubID string
174 var bugGithubURL string
175
176 // Special case:
177 // if a user try to export a bug that is not already exported to Github (or imported
178 // from Github) and we do not have the token of the bug author, there is nothing we can do.
179
180 // first operation is always createOp
181 createOp := snapshot.Operations[0].(*bug.CreateOperation)
182 author := snapshot.Author
183
184 // skip bug if origin is not allowed
185 origin, ok := snapshot.GetCreateMetadata(core.MetaKeyOrigin)
186 if ok && origin != target {
187 out <- core.NewExportNothing(b.Id(), fmt.Sprintf("issue tagged with origin: %s", origin))
188 return
189 }
190
191 // get github bug ID
192 githubID, ok := snapshot.GetCreateMetadata(metaKeyGithubId)
193 if ok {
194 githubURL, ok := snapshot.GetCreateMetadata(metaKeyGithubUrl)
195 if !ok {
196 // if we find github ID, github URL must be found too
197 err := fmt.Errorf("incomplete Github metadata: expected to find issue URL")
198 out <- core.NewExportError(err, b.Id())
199 }
200
201 // extract owner and project
202 owner, project, err := splitURL(githubURL)
203 if err != nil {
204 err := fmt.Errorf("bad project url: %v", err)
205 out <- core.NewExportError(err, b.Id())
206 return
207 }
208
209 // ignore issue coming from other repositories
210 if owner != ge.conf[keyOwner] && project != ge.conf[keyProject] {
211 out <- core.NewExportNothing(b.Id(), fmt.Sprintf("skipping issue from url:%s", githubURL))
212 return
213 }
214
215 // will be used to mark operation related to a bug as exported
216 bugGithubID = githubID
217 bugGithubURL = githubURL
218
219 } else {
220 // check that we have a token for operation author
221 client, err := ge.getClientForIdentity(author.Id())
222 if err != nil {
223 // if bug is still not exported and we do not have the author stop the execution
224 out <- core.NewExportNothing(b.Id(), fmt.Sprintf("missing author token"))
225 return
226 }
227
228 // create bug
229 id, url, err := createGithubIssue(ctx, client, ge.repositoryID, createOp.Title, createOp.Message)
230 if err != nil {
231 err := errors.Wrap(err, "exporting github issue")
232 out <- core.NewExportError(err, b.Id())
233 return
234 }
235
236 out <- core.NewExportBug(b.Id())
237
238 // mark bug creation operation as exported
239 if err := markOperationAsExported(b, createOp.Id(), id, url); err != nil {
240 err := errors.Wrap(err, "marking operation as exported")
241 out <- core.NewExportError(err, b.Id())
242 return
243 }
244
245 // commit operation to avoid creating multiple issues with multiple pushes
246 if err := b.CommitAsNeeded(); err != nil {
247 err := errors.Wrap(err, "bug commit")
248 out <- core.NewExportError(err, b.Id())
249 return
250 }
251
252 // cache bug github ID and URL
253 bugGithubID = id
254 bugGithubURL = url
255 }
256
257 // cache operation github id
258 ge.cachedOperationIDs[createOp.Id()] = bugGithubID
259
260 for _, op := range snapshot.Operations[1:] {
261 // ignore SetMetadata operations
262 if _, ok := op.(*bug.SetMetadataOperation); ok {
263 continue
264 }
265
266 // ignore operations already existing in github (due to import or export)
267 // cache the ID of already exported or imported issues and events from Github
268 if id, ok := op.GetMetadata(metaKeyGithubId); ok {
269 ge.cachedOperationIDs[op.Id()] = id
270 continue
271 }
272
273 opAuthor := op.GetAuthor()
274 client, err := ge.getClientForIdentity(opAuthor.Id())
275 if err != nil {
276 continue
277 }
278
279 var id, url string
280 switch op := op.(type) {
281 case *bug.AddCommentOperation:
282 // send operation to github
283 id, url, err = addCommentGithubIssue(ctx, client, bugGithubID, op.Message)
284 if err != nil {
285 err := errors.Wrap(err, "adding comment")
286 out <- core.NewExportError(err, b.Id())
287 return
288 }
289
290 out <- core.NewExportComment(op.Id())
291
292 // cache comment id
293 ge.cachedOperationIDs[op.Id()] = id
294
295 case *bug.EditCommentOperation:
296 // Since github doesn't consider the issue body as a comment
297 if op.Target == createOp.Id() {
298
299 // case bug creation operation: we need to edit the Github issue
300 if err := updateGithubIssueBody(ctx, client, bugGithubID, op.Message); err != nil {
301 err := errors.Wrap(err, "editing issue")
302 out <- core.NewExportError(err, b.Id())
303 return
304 }
305
306 out <- core.NewExportCommentEdition(op.Id())
307
308 id = bugGithubID
309 url = bugGithubURL
310
311 } else {
312
313 // case comment edition operation: we need to edit the Github comment
314 commentID, ok := ge.cachedOperationIDs[op.Target]
315 if !ok {
316 panic("unexpected error: comment id not found")
317 }
318
319 eid, eurl, err := editCommentGithubIssue(ctx, client, commentID, op.Message)
320 if err != nil {
321 err := errors.Wrap(err, "editing comment")
322 out <- core.NewExportError(err, b.Id())
323 return
324 }
325
326 out <- core.NewExportCommentEdition(op.Id())
327
328 // use comment id/url instead of issue id/url
329 id = eid
330 url = eurl
331 }
332
333 case *bug.SetStatusOperation:
334 if err := updateGithubIssueStatus(ctx, client, bugGithubID, op.Status); err != nil {
335 err := errors.Wrap(err, "editing status")
336 out <- core.NewExportError(err, b.Id())
337 return
338 }
339
340 out <- core.NewExportStatusChange(op.Id())
341
342 id = bugGithubID
343 url = bugGithubURL
344
345 case *bug.SetTitleOperation:
346 if err := updateGithubIssueTitle(ctx, client, bugGithubID, op.Title); err != nil {
347 err := errors.Wrap(err, "editing title")
348 out <- core.NewExportError(err, b.Id())
349 return
350 }
351
352 out <- core.NewExportTitleEdition(op.Id())
353
354 id = bugGithubID
355 url = bugGithubURL
356
357 case *bug.LabelChangeOperation:
358 if err := ge.updateGithubIssueLabels(ctx, client, bugGithubID, op.Added, op.Removed); err != nil {
359 err := errors.Wrap(err, "updating labels")
360 out <- core.NewExportError(err, b.Id())
361 return
362 }
363
364 out <- core.NewExportLabelChange(op.Id())
365
366 id = bugGithubID
367 url = bugGithubURL
368
369 default:
370 panic("unhandled operation type case")
371 }
372
373 // mark operation as exported
374 if err := markOperationAsExported(b, op.Id(), id, url); err != nil {
375 err := errors.Wrap(err, "marking operation as exported")
376 out <- core.NewExportError(err, b.Id())
377 return
378 }
379
380 // commit at each operation export to avoid exporting same events multiple times
381 if err := b.CommitAsNeeded(); err != nil {
382 err := errors.Wrap(err, "bug commit")
383 out <- core.NewExportError(err, b.Id())
384 return
385 }
386
387 bugUpdated = true
388 }
389
390 if !bugUpdated {
391 out <- core.NewExportNothing(b.Id(), "nothing has been exported")
392 }
393}
394
395// getRepositoryNodeID request github api v3 to get repository node id
396func getRepositoryNodeID(ctx context.Context, token *auth.Token, owner, project string) (string, error) {
397 url := fmt.Sprintf("%s/repos/%s/%s", githubV3Url, owner, project)
398 client := &http.Client{}
399
400 req, err := http.NewRequest("GET", url, nil)
401 if err != nil {
402 return "", err
403 }
404
405 // need the token for private repositories
406 req.Header.Set("Authorization", fmt.Sprintf("token %s", token.Value))
407
408 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
409 defer cancel()
410 req = req.WithContext(ctx)
411
412 resp, err := client.Do(req)
413 if err != nil {
414 return "", err
415 }
416
417 if resp.StatusCode != http.StatusOK {
418 return "", fmt.Errorf("HTTP error %v retrieving repository node id", resp.StatusCode)
419 }
420
421 aux := struct {
422 NodeID string `json:"node_id"`
423 }{}
424
425 data, _ := ioutil.ReadAll(resp.Body)
426 err = resp.Body.Close()
427 if err != nil {
428 return "", err
429 }
430
431 err = json.Unmarshal(data, &aux)
432 if err != nil {
433 return "", err
434 }
435
436 return aux.NodeID, nil
437}
438
439func markOperationAsExported(b *cache.BugCache, target entity.Id, githubID, githubURL string) error {
440 _, err := b.SetMetadata(
441 target,
442 map[string]string{
443 metaKeyGithubId: githubID,
444 metaKeyGithubUrl: githubURL,
445 },
446 )
447
448 return err
449}
450
451func (ge *githubExporter) cacheGithubLabels(ctx context.Context, gc *githubv4.Client) error {
452 variables := map[string]interface{}{
453 "owner": githubv4.String(ge.conf[keyOwner]),
454 "name": githubv4.String(ge.conf[keyProject]),
455 "first": githubv4.Int(10),
456 "after": (*githubv4.String)(nil),
457 }
458
459 q := labelsQuery{}
460
461 hasNextPage := true
462 for hasNextPage {
463 // create a new timeout context at each iteration
464 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
465 defer cancel()
466
467 if err := gc.Query(ctx, &q, variables); err != nil {
468 return err
469 }
470
471 for _, label := range q.Repository.Labels.Nodes {
472 ge.cachedLabels[label.Name] = label.ID
473 }
474
475 hasNextPage = q.Repository.Labels.PageInfo.HasNextPage
476 variables["after"] = q.Repository.Labels.PageInfo.EndCursor
477 }
478
479 return nil
480}
481
482func (ge *githubExporter) getLabelID(gc *githubv4.Client, label string) (string, error) {
483 label = strings.ToLower(label)
484 for cachedLabel, ID := range ge.cachedLabels {
485 if label == strings.ToLower(cachedLabel) {
486 return ID, nil
487 }
488 }
489
490 return "", fmt.Errorf("didn't find label id in cache")
491}
492
493// create a new label and return it github id
494// NOTE: since createLabel mutation is still in preview mode we use github api v3 to create labels
495// see https://developer.github.com/v4/mutation/createlabel/ and https://developer.github.com/v4/previews/#labels-preview
496func (ge *githubExporter) createGithubLabel(ctx context.Context, label, color string) (string, error) {
497 url := fmt.Sprintf("%s/repos/%s/%s/labels", githubV3Url, ge.conf[keyOwner], ge.conf[keyProject])
498 client := &http.Client{}
499
500 params := struct {
501 Name string `json:"name"`
502 Color string `json:"color"`
503 Description string `json:"description"`
504 }{
505 Name: label,
506 Color: color,
507 }
508
509 data, err := json.Marshal(params)
510 if err != nil {
511 return "", err
512 }
513
514 req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
515 if err != nil {
516 return "", err
517 }
518
519 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
520 defer cancel()
521 req = req.WithContext(ctx)
522
523 // need the token for private repositories
524 req.Header.Set("Authorization", fmt.Sprintf("token %s", ge.defaultToken.Value))
525
526 resp, err := client.Do(req)
527 if err != nil {
528 return "", err
529 }
530
531 if resp.StatusCode != http.StatusCreated {
532 return "", fmt.Errorf("error creating label: response status %v", resp.StatusCode)
533 }
534
535 aux := struct {
536 ID int `json:"id"`
537 NodeID string `json:"node_id"`
538 Color string `json:"color"`
539 }{}
540
541 data, _ = ioutil.ReadAll(resp.Body)
542 defer resp.Body.Close()
543
544 err = json.Unmarshal(data, &aux)
545 if err != nil {
546 return "", err
547 }
548
549 return aux.NodeID, nil
550}
551
552/**
553// create github label using api v4
554func (ge *githubExporter) createGithubLabelV4(gc *githubv4.Client, label, labelColor string) (string, error) {
555 m := createLabelMutation{}
556 input := createLabelInput{
557 RepositoryID: ge.repositoryID,
558 Name: githubv4.String(label),
559 Color: githubv4.String(labelColor),
560 }
561
562 parentCtx := context.Background()
563 ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout)
564 defer cancel()
565
566 if err := gc.Mutate(ctx, &m, input, nil); err != nil {
567 return "", err
568 }
569
570 return m.CreateLabel.Label.ID, nil
571}
572*/
573
574func (ge *githubExporter) getOrCreateGithubLabelID(ctx context.Context, gc *githubv4.Client, repositoryID string, label bug.Label) (string, error) {
575 // try to get label id from cache
576 labelID, err := ge.getLabelID(gc, string(label))
577 if err == nil {
578 return labelID, nil
579 }
580
581 // RGBA to hex color
582 rgba := label.Color().RGBA()
583 hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B)
584
585 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
586 defer cancel()
587
588 labelID, err = ge.createGithubLabel(ctx, string(label), hexColor)
589 if err != nil {
590 return "", err
591 }
592
593 return labelID, nil
594}
595
596func (ge *githubExporter) getLabelsIDs(ctx context.Context, gc *githubv4.Client, repositoryID string, labels []bug.Label) ([]githubv4.ID, error) {
597 ids := make([]githubv4.ID, 0, len(labels))
598 var err error
599
600 // check labels ids
601 for _, label := range labels {
602 id, ok := ge.cachedLabels[string(label)]
603 if !ok {
604 // try to query label id
605 id, err = ge.getOrCreateGithubLabelID(ctx, gc, repositoryID, label)
606 if err != nil {
607 return nil, errors.Wrap(err, "get or create github label")
608 }
609
610 // cache label id
611 ge.cachedLabels[string(label)] = id
612 }
613
614 ids = append(ids, githubv4.ID(id))
615 }
616
617 return ids, nil
618}
619
620// create a github issue and return it ID
621func createGithubIssue(ctx context.Context, gc *githubv4.Client, repositoryID, title, body string) (string, string, error) {
622 m := &createIssueMutation{}
623 input := githubv4.CreateIssueInput{
624 RepositoryID: repositoryID,
625 Title: githubv4.String(title),
626 Body: (*githubv4.String)(&body),
627 }
628
629 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
630 defer cancel()
631
632 if err := gc.Mutate(ctx, m, input, nil); err != nil {
633 return "", "", err
634 }
635
636 issue := m.CreateIssue.Issue
637 return issue.ID, issue.URL, nil
638}
639
640// add a comment to an issue and return it ID
641func addCommentGithubIssue(ctx context.Context, gc *githubv4.Client, subjectID string, body string) (string, string, error) {
642 m := &addCommentToIssueMutation{}
643 input := githubv4.AddCommentInput{
644 SubjectID: subjectID,
645 Body: githubv4.String(body),
646 }
647
648 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
649 defer cancel()
650
651 if err := gc.Mutate(ctx, m, input, nil); err != nil {
652 return "", "", err
653 }
654
655 node := m.AddComment.CommentEdge.Node
656 return node.ID, node.URL, nil
657}
658
659func editCommentGithubIssue(ctx context.Context, gc *githubv4.Client, commentID, body string) (string, string, error) {
660 m := &updateIssueCommentMutation{}
661 input := githubv4.UpdateIssueCommentInput{
662 ID: commentID,
663 Body: githubv4.String(body),
664 }
665
666 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
667 defer cancel()
668
669 if err := gc.Mutate(ctx, m, input, nil); err != nil {
670 return "", "", err
671 }
672
673 return commentID, m.UpdateIssueComment.IssueComment.URL, nil
674}
675
676func updateGithubIssueStatus(ctx context.Context, gc *githubv4.Client, id string, status bug.Status) error {
677 m := &updateIssueMutation{}
678
679 // set state
680 var state githubv4.IssueState
681
682 switch status {
683 case bug.OpenStatus:
684 state = githubv4.IssueStateOpen
685 case bug.ClosedStatus:
686 state = githubv4.IssueStateClosed
687 default:
688 panic("unknown bug state")
689 }
690
691 input := githubv4.UpdateIssueInput{
692 ID: id,
693 State: &state,
694 }
695
696 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
697 defer cancel()
698
699 if err := gc.Mutate(ctx, m, input, nil); err != nil {
700 return err
701 }
702
703 return nil
704}
705
706func updateGithubIssueBody(ctx context.Context, gc *githubv4.Client, id string, body string) error {
707 m := &updateIssueMutation{}
708 input := githubv4.UpdateIssueInput{
709 ID: id,
710 Body: (*githubv4.String)(&body),
711 }
712
713 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
714 defer cancel()
715
716 if err := gc.Mutate(ctx, m, input, nil); err != nil {
717 return err
718 }
719
720 return nil
721}
722
723func updateGithubIssueTitle(ctx context.Context, gc *githubv4.Client, id, title string) error {
724 m := &updateIssueMutation{}
725 input := githubv4.UpdateIssueInput{
726 ID: id,
727 Title: (*githubv4.String)(&title),
728 }
729
730 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
731 defer cancel()
732
733 if err := gc.Mutate(ctx, m, input, nil); err != nil {
734 return err
735 }
736
737 return nil
738}
739
740// update github issue labels
741func (ge *githubExporter) updateGithubIssueLabels(ctx context.Context, gc *githubv4.Client, labelableID string, added, removed []bug.Label) error {
742 reqCtx, cancel := context.WithTimeout(ctx, defaultTimeout)
743 defer cancel()
744
745 wg, ctx := errgroup.WithContext(ctx)
746 if len(added) > 0 {
747 wg.Go(func() error {
748 addedIDs, err := ge.getLabelsIDs(ctx, gc, labelableID, added)
749 if err != nil {
750 return err
751 }
752
753 m := &addLabelsToLabelableMutation{}
754 inputAdd := githubv4.AddLabelsToLabelableInput{
755 LabelableID: labelableID,
756 LabelIDs: addedIDs,
757 }
758
759 // add labels
760 if err := gc.Mutate(reqCtx, m, inputAdd, nil); err != nil {
761 return err
762 }
763 return nil
764 })
765 }
766
767 if len(removed) > 0 {
768 wg.Go(func() error {
769 removedIDs, err := ge.getLabelsIDs(ctx, gc, labelableID, removed)
770 if err != nil {
771 return err
772 }
773
774 m2 := &removeLabelsFromLabelableMutation{}
775 inputRemove := githubv4.RemoveLabelsFromLabelableInput{
776 LabelableID: labelableID,
777 LabelIDs: removedIDs,
778 }
779
780 // remove label labels
781 if err := gc.Mutate(reqCtx, m2, inputRemove, nil); err != nil {
782 return err
783 }
784 return nil
785 })
786 }
787
788 return wg.Wait()
789}