1package github
2
3import (
4 "bytes"
5 "context"
6 "encoding/json"
7 "fmt"
8 "io/ioutil"
9 "net/http"
10 "strings"
11 "time"
12
13 "github.com/pkg/errors"
14 "github.com/shurcooL/githubv4"
15 "golang.org/x/sync/errgroup"
16
17 "github.com/MichaelMure/git-bug/bridge/core"
18 "github.com/MichaelMure/git-bug/bug"
19 "github.com/MichaelMure/git-bug/cache"
20 "github.com/MichaelMure/git-bug/entity"
21)
22
23var (
24 ErrMissingIdentityToken = errors.New("missing identity token")
25)
26
27// githubExporter implement the Exporter interface
28type githubExporter struct {
29 conf core.Configuration
30
31 // cache identities clients
32 identityClient map[entity.Id]*githubv4.Client
33
34 // map identities with their tokens
35 identityToken map[entity.Id]string
36
37 // github repository ID
38 repositoryID string
39
40 // cache identifiers used to speed up exporting operations
41 // cleared for each bug
42 cachedOperationIDs map[entity.Id]string
43
44 // cache labels used to speed up exporting labels events
45 cachedLabels map[string]string
46}
47
48// Init .
49func (ge *githubExporter) Init(conf core.Configuration) error {
50 ge.conf = conf
51 //TODO: initialize with multiple tokens
52 ge.identityToken = make(map[entity.Id]string)
53 ge.identityClient = make(map[entity.Id]*githubv4.Client)
54 ge.cachedOperationIDs = make(map[entity.Id]string)
55 ge.cachedLabels = make(map[string]string)
56 return nil
57}
58
59// getIdentityClient return a githubv4 API client configured with the access token of the given identity.
60// if no client were found it will initialize it from the known tokens map and cache it for next use
61func (ge *githubExporter) getIdentityClient(id entity.Id) (*githubv4.Client, error) {
62 client, ok := ge.identityClient[id]
63 if ok {
64 return client, nil
65 }
66
67 // get token
68 token, ok := ge.identityToken[id]
69 if !ok {
70 return nil, ErrMissingIdentityToken
71 }
72
73 // create client
74 client = buildClient(token)
75 // cache client
76 ge.identityClient[id] = client
77
78 return client, nil
79}
80
81// ExportAll export all event made by the current user to Github
82func (ge *githubExporter) ExportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ExportResult, error) {
83 out := make(chan core.ExportResult)
84
85 user, err := repo.GetUserIdentity()
86 if err != nil {
87 return nil, err
88 }
89
90 ge.identityToken[user.Id()] = ge.conf[core.ConfigKeyToken]
91
92 // get repository node id
93 ge.repositoryID, err = getRepositoryNodeID(
94 ctx,
95 ge.conf[keyOwner],
96 ge.conf[keyProject],
97 ge.conf[core.ConfigKeyToken],
98 )
99
100 if err != nil {
101 return nil, err
102 }
103
104 client, err := ge.getIdentityClient(user.Id())
105 if err != nil {
106 return nil, err
107 }
108
109 // query all labels
110 err = ge.cacheGithubLabels(ctx, client)
111 if err != nil {
112 return nil, err
113 }
114
115 go func() {
116 defer close(out)
117
118 var allIdentitiesIds []entity.Id
119 for id := range ge.identityToken {
120 allIdentitiesIds = append(allIdentitiesIds, id)
121 }
122
123 allBugsIds := repo.AllBugsIds()
124
125 for _, id := range allBugsIds {
126 b, err := repo.ResolveBug(id)
127 if err != nil {
128 out <- core.NewExportError(errors.Wrap(err, "can't load bug"), id)
129 return
130 }
131
132 select {
133
134 case <-ctx.Done():
135 // stop iterating if context cancel function is called
136 return
137
138 default:
139 snapshot := b.Snapshot()
140
141 // ignore issues created before since date
142 // TODO: compare the Lamport time instead of using the unix time
143 if snapshot.CreatedAt.Before(since) {
144 out <- core.NewExportNothing(b.Id(), "bug created before the since date")
145 continue
146 }
147
148 if snapshot.HasAnyActor(allIdentitiesIds...) {
149 // try to export the bug and it associated events
150 ge.exportBug(ctx, b, since, out)
151 }
152 }
153 }
154 }()
155
156 return out, nil
157}
158
159// exportBug publish bugs and related events
160func (ge *githubExporter) exportBug(ctx context.Context, b *cache.BugCache, since time.Time, out chan<- core.ExportResult) {
161 snapshot := b.Snapshot()
162 var bugUpdated bool
163
164 var bugGithubID string
165 var bugGithubURL string
166
167 // Special case:
168 // if a user try to export a bug that is not already exported to Github (or imported
169 // from Github) and we do not have the token of the bug author, there is nothing we can do.
170
171 // first operation is always createOp
172 createOp := snapshot.Operations[0].(*bug.CreateOperation)
173 author := snapshot.Author
174
175 // skip bug if origin is not allowed
176 origin, ok := snapshot.GetCreateMetadata(core.MetaKeyOrigin)
177 if ok && origin != target {
178 out <- core.NewExportNothing(b.Id(), fmt.Sprintf("issue tagged with origin: %s", origin))
179 return
180 }
181
182 // get github bug ID
183 githubID, ok := snapshot.GetCreateMetadata(metaKeyGithubId)
184 if ok {
185 githubURL, ok := snapshot.GetCreateMetadata(metaKeyGithubUrl)
186 if !ok {
187 // if we find github ID, github URL must be found too
188 err := fmt.Errorf("incomplete Github metadata: expected to find issue URL")
189 out <- core.NewExportError(err, b.Id())
190 }
191
192 // extract owner and project
193 owner, project, err := splitURL(githubURL)
194 if err != nil {
195 err := fmt.Errorf("bad project url: %v", err)
196 out <- core.NewExportError(err, b.Id())
197 return
198 }
199
200 // ignore issue coming from other repositories
201 if owner != ge.conf[keyOwner] && project != ge.conf[keyProject] {
202 out <- core.NewExportNothing(b.Id(), fmt.Sprintf("skipping issue from url:%s", githubURL))
203 return
204 }
205
206 // will be used to mark operation related to a bug as exported
207 bugGithubID = githubID
208 bugGithubURL = githubURL
209
210 } else {
211 // check that we have a token for operation author
212 client, err := ge.getIdentityClient(author.Id())
213 if err != nil {
214 // if bug is still not exported and we do not have the author stop the execution
215 out <- core.NewExportNothing(b.Id(), fmt.Sprintf("missing author token"))
216 return
217 }
218
219 // create bug
220 id, url, err := createGithubIssue(ctx, client, ge.repositoryID, createOp.Title, createOp.Message)
221 if err != nil {
222 err := errors.Wrap(err, "exporting github issue")
223 out <- core.NewExportError(err, b.Id())
224 return
225 }
226
227 out <- core.NewExportBug(b.Id())
228
229 // mark bug creation operation as exported
230 if err := markOperationAsExported(b, createOp.Id(), id, url); err != nil {
231 err := errors.Wrap(err, "marking operation as exported")
232 out <- core.NewExportError(err, b.Id())
233 return
234 }
235
236 // commit operation to avoid creating multiple issues with multiple pushes
237 if err := b.CommitAsNeeded(); err != nil {
238 err := errors.Wrap(err, "bug commit")
239 out <- core.NewExportError(err, b.Id())
240 return
241 }
242
243 // cache bug github ID and URL
244 bugGithubID = id
245 bugGithubURL = url
246 }
247
248 // cache operation github id
249 ge.cachedOperationIDs[createOp.Id()] = bugGithubID
250
251 for _, op := range snapshot.Operations[1:] {
252 // ignore SetMetadata operations
253 if _, ok := op.(*bug.SetMetadataOperation); ok {
254 continue
255 }
256
257 // ignore operations already existing in github (due to import or export)
258 // cache the ID of already exported or imported issues and events from Github
259 if id, ok := op.GetMetadata(metaKeyGithubId); ok {
260 ge.cachedOperationIDs[op.Id()] = id
261 continue
262 }
263
264 opAuthor := op.GetAuthor()
265 client, err := ge.getIdentityClient(opAuthor.Id())
266 if err != nil {
267 continue
268 }
269
270 var id, url string
271 switch op := op.(type) {
272 case *bug.AddCommentOperation:
273 // send operation to github
274 id, url, err = addCommentGithubIssue(ctx, client, bugGithubID, op.Message)
275 if err != nil {
276 err := errors.Wrap(err, "adding comment")
277 out <- core.NewExportError(err, b.Id())
278 return
279 }
280
281 out <- core.NewExportComment(op.Id())
282
283 // cache comment id
284 ge.cachedOperationIDs[op.Id()] = id
285
286 case *bug.EditCommentOperation:
287 // Since github doesn't consider the issue body as a comment
288 if op.Target == createOp.Id() {
289
290 // case bug creation operation: we need to edit the Github issue
291 if err := updateGithubIssueBody(ctx, client, bugGithubID, op.Message); err != nil {
292 err := errors.Wrap(err, "editing issue")
293 out <- core.NewExportError(err, b.Id())
294 return
295 }
296
297 out <- core.NewExportCommentEdition(op.Id())
298
299 id = bugGithubID
300 url = bugGithubURL
301
302 } else {
303
304 // case comment edition operation: we need to edit the Github comment
305 commentID, ok := ge.cachedOperationIDs[op.Target]
306 if !ok {
307 panic("unexpected error: comment id not found")
308 }
309
310 eid, eurl, err := editCommentGithubIssue(ctx, client, commentID, op.Message)
311 if err != nil {
312 err := errors.Wrap(err, "editing comment")
313 out <- core.NewExportError(err, b.Id())
314 return
315 }
316
317 out <- core.NewExportCommentEdition(op.Id())
318
319 // use comment id/url instead of issue id/url
320 id = eid
321 url = eurl
322 }
323
324 case *bug.SetStatusOperation:
325 if err := updateGithubIssueStatus(ctx, client, bugGithubID, op.Status); err != nil {
326 err := errors.Wrap(err, "editing status")
327 out <- core.NewExportError(err, b.Id())
328 return
329 }
330
331 out <- core.NewExportStatusChange(op.Id())
332
333 id = bugGithubID
334 url = bugGithubURL
335
336 case *bug.SetTitleOperation:
337 if err := updateGithubIssueTitle(ctx, client, bugGithubID, op.Title); err != nil {
338 err := errors.Wrap(err, "editing title")
339 out <- core.NewExportError(err, b.Id())
340 return
341 }
342
343 out <- core.NewExportTitleEdition(op.Id())
344
345 id = bugGithubID
346 url = bugGithubURL
347
348 case *bug.LabelChangeOperation:
349 if err := ge.updateGithubIssueLabels(ctx, client, bugGithubID, op.Added, op.Removed); err != nil {
350 err := errors.Wrap(err, "updating labels")
351 out <- core.NewExportError(err, b.Id())
352 return
353 }
354
355 out <- core.NewExportLabelChange(op.Id())
356
357 id = bugGithubID
358 url = bugGithubURL
359
360 default:
361 panic("unhandled operation type case")
362 }
363
364 // mark operation as exported
365 if err := markOperationAsExported(b, op.Id(), id, url); err != nil {
366 err := errors.Wrap(err, "marking operation as exported")
367 out <- core.NewExportError(err, b.Id())
368 return
369 }
370
371 // commit at each operation export to avoid exporting same events multiple times
372 if err := b.CommitAsNeeded(); err != nil {
373 err := errors.Wrap(err, "bug commit")
374 out <- core.NewExportError(err, b.Id())
375 return
376 }
377
378 bugUpdated = true
379 }
380
381 if !bugUpdated {
382 out <- core.NewExportNothing(b.Id(), "nothing has been exported")
383 }
384}
385
386// getRepositoryNodeID request github api v3 to get repository node id
387func getRepositoryNodeID(ctx context.Context, owner, project, token string) (string, error) {
388 url := fmt.Sprintf("%s/repos/%s/%s", githubV3Url, owner, project)
389 client := &http.Client{}
390
391 req, err := http.NewRequest("GET", url, nil)
392 if err != nil {
393 return "", err
394 }
395
396 // need the token for private repositories
397 req.Header.Set("Authorization", fmt.Sprintf("token %s", token))
398
399 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
400 defer cancel()
401 req = req.WithContext(ctx)
402
403 resp, err := client.Do(req)
404 if err != nil {
405 return "", err
406 }
407
408 if resp.StatusCode != http.StatusOK {
409 return "", fmt.Errorf("HTTP error %v retrieving repository node id", resp.StatusCode)
410 }
411
412 aux := struct {
413 NodeID string `json:"node_id"`
414 }{}
415
416 data, _ := ioutil.ReadAll(resp.Body)
417 err = resp.Body.Close()
418 if err != nil {
419 return "", err
420 }
421
422 err = json.Unmarshal(data, &aux)
423 if err != nil {
424 return "", err
425 }
426
427 return aux.NodeID, nil
428}
429
430func markOperationAsExported(b *cache.BugCache, target entity.Id, githubID, githubURL string) error {
431 _, err := b.SetMetadata(
432 target,
433 map[string]string{
434 metaKeyGithubId: githubID,
435 metaKeyGithubUrl: githubURL,
436 },
437 )
438
439 return err
440}
441
442func (ge *githubExporter) cacheGithubLabels(ctx context.Context, gc *githubv4.Client) error {
443 variables := map[string]interface{}{
444 "owner": githubv4.String(ge.conf[keyOwner]),
445 "name": githubv4.String(ge.conf[keyProject]),
446 "first": githubv4.Int(10),
447 "after": (*githubv4.String)(nil),
448 }
449
450 q := labelsQuery{}
451
452 hasNextPage := true
453 for hasNextPage {
454 // create a new timeout context at each iteration
455 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
456 defer cancel()
457
458 if err := gc.Query(ctx, &q, variables); err != nil {
459 return err
460 }
461
462 for _, label := range q.Repository.Labels.Nodes {
463 ge.cachedLabels[label.Name] = label.ID
464 }
465
466 hasNextPage = q.Repository.Labels.PageInfo.HasNextPage
467 variables["after"] = q.Repository.Labels.PageInfo.EndCursor
468 }
469
470 return nil
471}
472
473func (ge *githubExporter) getLabelID(gc *githubv4.Client, label string) (string, error) {
474 label = strings.ToLower(label)
475 for cachedLabel, ID := range ge.cachedLabels {
476 if label == strings.ToLower(cachedLabel) {
477 return ID, nil
478 }
479 }
480
481 return "", fmt.Errorf("didn't find label id in cache")
482}
483
484// create a new label and return it github id
485// NOTE: since createLabel mutation is still in preview mode we use github api v3 to create labels
486// see https://developer.github.com/v4/mutation/createlabel/ and https://developer.github.com/v4/previews/#labels-preview
487func (ge *githubExporter) createGithubLabel(ctx context.Context, label, color string) (string, error) {
488 url := fmt.Sprintf("%s/repos/%s/%s/labels", githubV3Url, ge.conf[keyOwner], ge.conf[keyProject])
489 client := &http.Client{}
490
491 params := struct {
492 Name string `json:"name"`
493 Color string `json:"color"`
494 Description string `json:"description"`
495 }{
496 Name: label,
497 Color: color,
498 }
499
500 data, err := json.Marshal(params)
501 if err != nil {
502 return "", err
503 }
504
505 req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
506 if err != nil {
507 return "", err
508 }
509
510 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
511 defer cancel()
512 req = req.WithContext(ctx)
513
514 // need the token for private repositories
515 req.Header.Set("Authorization", fmt.Sprintf("token %s", ge.conf[core.ConfigKeyToken]))
516
517 resp, err := client.Do(req)
518 if err != nil {
519 return "", err
520 }
521
522 if resp.StatusCode != http.StatusCreated {
523 return "", fmt.Errorf("error creating label: response status %v", resp.StatusCode)
524 }
525
526 aux := struct {
527 ID int `json:"id"`
528 NodeID string `json:"node_id"`
529 Color string `json:"color"`
530 }{}
531
532 data, _ = ioutil.ReadAll(resp.Body)
533 defer resp.Body.Close()
534
535 err = json.Unmarshal(data, &aux)
536 if err != nil {
537 return "", err
538 }
539
540 return aux.NodeID, nil
541}
542
543/**
544// create github label using api v4
545func (ge *githubExporter) createGithubLabelV4(gc *githubv4.Client, label, labelColor string) (string, error) {
546 m := createLabelMutation{}
547 input := createLabelInput{
548 RepositoryID: ge.repositoryID,
549 Name: githubv4.String(label),
550 Color: githubv4.String(labelColor),
551 }
552
553 parentCtx := context.Background()
554 ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout)
555 defer cancel()
556
557 if err := gc.Mutate(ctx, &m, input, nil); err != nil {
558 return "", err
559 }
560
561 return m.CreateLabel.Label.ID, nil
562}
563*/
564
565func (ge *githubExporter) getOrCreateGithubLabelID(ctx context.Context, gc *githubv4.Client, repositoryID string, label bug.Label) (string, error) {
566 // try to get label id from cache
567 labelID, err := ge.getLabelID(gc, string(label))
568 if err == nil {
569 return labelID, nil
570 }
571
572 // RGBA to hex color
573 rgba := label.Color().RGBA()
574 hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B)
575
576 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
577 defer cancel()
578
579 labelID, err = ge.createGithubLabel(ctx, string(label), hexColor)
580 if err != nil {
581 return "", err
582 }
583
584 return labelID, nil
585}
586
587func (ge *githubExporter) getLabelsIDs(ctx context.Context, gc *githubv4.Client, repositoryID string, labels []bug.Label) ([]githubv4.ID, error) {
588 ids := make([]githubv4.ID, 0, len(labels))
589 var err error
590
591 // check labels ids
592 for _, label := range labels {
593 id, ok := ge.cachedLabels[string(label)]
594 if !ok {
595 // try to query label id
596 id, err = ge.getOrCreateGithubLabelID(ctx, gc, repositoryID, label)
597 if err != nil {
598 return nil, errors.Wrap(err, "get or create github label")
599 }
600
601 // cache label id
602 ge.cachedLabels[string(label)] = id
603 }
604
605 ids = append(ids, githubv4.ID(id))
606 }
607
608 return ids, nil
609}
610
611// create a github issue and return it ID
612func createGithubIssue(ctx context.Context, gc *githubv4.Client, repositoryID, title, body string) (string, string, error) {
613 m := &createIssueMutation{}
614 input := githubv4.CreateIssueInput{
615 RepositoryID: repositoryID,
616 Title: githubv4.String(title),
617 Body: (*githubv4.String)(&body),
618 }
619
620 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
621 defer cancel()
622
623 if err := gc.Mutate(ctx, m, input, nil); err != nil {
624 return "", "", err
625 }
626
627 issue := m.CreateIssue.Issue
628 return issue.ID, issue.URL, nil
629}
630
631// add a comment to an issue and return it ID
632func addCommentGithubIssue(ctx context.Context, gc *githubv4.Client, subjectID string, body string) (string, string, error) {
633 m := &addCommentToIssueMutation{}
634 input := githubv4.AddCommentInput{
635 SubjectID: subjectID,
636 Body: githubv4.String(body),
637 }
638
639 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
640 defer cancel()
641
642 if err := gc.Mutate(ctx, m, input, nil); err != nil {
643 return "", "", err
644 }
645
646 node := m.AddComment.CommentEdge.Node
647 return node.ID, node.URL, nil
648}
649
650func editCommentGithubIssue(ctx context.Context, gc *githubv4.Client, commentID, body string) (string, string, error) {
651 m := &updateIssueCommentMutation{}
652 input := githubv4.UpdateIssueCommentInput{
653 ID: commentID,
654 Body: githubv4.String(body),
655 }
656
657 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
658 defer cancel()
659
660 if err := gc.Mutate(ctx, m, input, nil); err != nil {
661 return "", "", err
662 }
663
664 return commentID, m.UpdateIssueComment.IssueComment.URL, nil
665}
666
667func updateGithubIssueStatus(ctx context.Context, gc *githubv4.Client, id string, status bug.Status) error {
668 m := &updateIssueMutation{}
669
670 // set state
671 var state githubv4.IssueState
672
673 switch status {
674 case bug.OpenStatus:
675 state = githubv4.IssueStateOpen
676 case bug.ClosedStatus:
677 state = githubv4.IssueStateClosed
678 default:
679 panic("unknown bug state")
680 }
681
682 input := githubv4.UpdateIssueInput{
683 ID: id,
684 State: &state,
685 }
686
687 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
688 defer cancel()
689
690 if err := gc.Mutate(ctx, m, input, nil); err != nil {
691 return err
692 }
693
694 return nil
695}
696
697func updateGithubIssueBody(ctx context.Context, gc *githubv4.Client, id string, body string) error {
698 m := &updateIssueMutation{}
699 input := githubv4.UpdateIssueInput{
700 ID: id,
701 Body: (*githubv4.String)(&body),
702 }
703
704 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
705 defer cancel()
706
707 if err := gc.Mutate(ctx, m, input, nil); err != nil {
708 return err
709 }
710
711 return nil
712}
713
714func updateGithubIssueTitle(ctx context.Context, gc *githubv4.Client, id, title string) error {
715 m := &updateIssueMutation{}
716 input := githubv4.UpdateIssueInput{
717 ID: id,
718 Title: (*githubv4.String)(&title),
719 }
720
721 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
722 defer cancel()
723
724 if err := gc.Mutate(ctx, m, input, nil); err != nil {
725 return err
726 }
727
728 return nil
729}
730
731// update github issue labels
732func (ge *githubExporter) updateGithubIssueLabels(ctx context.Context, gc *githubv4.Client, labelableID string, added, removed []bug.Label) error {
733 reqCtx, cancel := context.WithTimeout(ctx, defaultTimeout)
734 defer cancel()
735
736 wg, ctx := errgroup.WithContext(ctx)
737 if len(added) > 0 {
738 wg.Go(func() error {
739 addedIDs, err := ge.getLabelsIDs(ctx, gc, labelableID, added)
740 if err != nil {
741 return err
742 }
743
744 m := &addLabelsToLabelableMutation{}
745 inputAdd := githubv4.AddLabelsToLabelableInput{
746 LabelableID: labelableID,
747 LabelIDs: addedIDs,
748 }
749
750 // add labels
751 if err := gc.Mutate(reqCtx, m, inputAdd, nil); err != nil {
752 return err
753 }
754 return nil
755 })
756 }
757
758 if len(removed) > 0 {
759 wg.Go(func() error {
760 removedIDs, err := ge.getLabelsIDs(ctx, gc, labelableID, removed)
761 if err != nil {
762 return err
763 }
764
765 m2 := &removeLabelsFromLabelableMutation{}
766 inputRemove := githubv4.RemoveLabelsFromLabelableInput{
767 LabelableID: labelableID,
768 LabelIDs: removedIDs,
769 }
770
771 // remove label labels
772 if err := gc.Mutate(reqCtx, m2, inputRemove, nil); err != nil {
773 return err
774 }
775 return nil
776 })
777 }
778
779 return wg.Wait()
780}