1package github
2
3import (
4 "bytes"
5 "context"
6 "encoding/json"
7 "fmt"
8 "io/ioutil"
9 "net/http"
10 "strings"
11 "time"
12
13 "github.com/pkg/errors"
14 "github.com/shurcooL/githubv4"
15 "golang.org/x/sync/errgroup"
16
17 "github.com/MichaelMure/git-bug/bridge/core"
18 "github.com/MichaelMure/git-bug/bug"
19 "github.com/MichaelMure/git-bug/cache"
20 "github.com/MichaelMure/git-bug/entity"
21)
22
23var (
24 ErrMissingIdentityToken = errors.New("missing identity token")
25)
26
27// githubExporter implement the Exporter interface
28type githubExporter struct {
29 conf core.Configuration
30
31 // cache identities clients
32 identityClient map[entity.Id]*githubv4.Client
33
34 // map identities with their tokens
35 identityToken map[entity.Id]string
36
37 // github repository ID
38 repositoryID string
39
40 // cache identifiers used to speed up exporting operations
41 // cleared for each bug
42 cachedOperationIDs map[entity.Id]string
43
44 // cache labels used to speed up exporting labels events
45 cachedLabels map[string]string
46}
47
48// Init .
49func (ge *githubExporter) Init(conf core.Configuration) error {
50 ge.conf = conf
51 //TODO: initialize with multiple tokens
52 ge.identityToken = make(map[entity.Id]string)
53 ge.identityClient = make(map[entity.Id]*githubv4.Client)
54 ge.cachedOperationIDs = make(map[entity.Id]string)
55 ge.cachedLabels = make(map[string]string)
56 return nil
57}
58
59// getIdentityClient return a githubv4 API client configured with the access token of the given identity.
60// if no client were found it will initialize it from the known tokens map and cache it for next use
61func (ge *githubExporter) getIdentityClient(id entity.Id) (*githubv4.Client, error) {
62 client, ok := ge.identityClient[id]
63 if ok {
64 return client, nil
65 }
66
67 // get token
68 token, ok := ge.identityToken[id]
69 if !ok {
70 return nil, ErrMissingIdentityToken
71 }
72
73 // create client
74 client = buildClient(token)
75 // cache client
76 ge.identityClient[id] = client
77
78 return client, nil
79}
80
81// ExportAll export all event made by the current user to Github
82func (ge *githubExporter) ExportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ExportResult, error) {
83 out := make(chan core.ExportResult)
84
85 user, err := repo.GetUserIdentity()
86 if err != nil {
87 return nil, err
88 }
89
90 ge.identityToken[user.Id()] = ge.conf[keyToken]
91
92 // get repository node id
93 ge.repositoryID, err = getRepositoryNodeID(
94 ctx,
95 ge.conf[keyOwner],
96 ge.conf[keyProject],
97 ge.conf[keyToken],
98 )
99
100 if err != nil {
101 return nil, err
102 }
103
104 client, err := ge.getIdentityClient(user.Id())
105 if err != nil {
106 return nil, err
107 }
108
109 // query all labels
110 err = ge.cacheGithubLabels(ctx, client)
111 if err != nil {
112 return nil, err
113 }
114
115 go func() {
116 defer close(out)
117
118 var allIdentitiesIds []entity.Id
119 for id := range ge.identityToken {
120 allIdentitiesIds = append(allIdentitiesIds, id)
121 }
122
123 allBugsIds := repo.AllBugsIds()
124
125 for _, id := range allBugsIds {
126 b, err := repo.ResolveBug(id)
127 if err != nil {
128 out <- core.NewExportError(errors.Wrap(err, "can't load bug"), id)
129 return
130 }
131
132 select {
133
134 case <-ctx.Done():
135 // stop iterating if context cancel function is called
136 return
137
138 default:
139 snapshot := b.Snapshot()
140
141 // ignore issues created before since date
142 // TODO: compare the Lamport time instead of using the unix time
143 if snapshot.CreatedAt.Before(since) {
144 out <- core.NewExportNothing(b.Id(), "bug created before the since date")
145 continue
146 }
147
148 if snapshot.HasAnyActor(allIdentitiesIds...) {
149 // try to export the bug and it associated events
150 ge.exportBug(ctx, b, since, out)
151 } else {
152 out <- core.NewExportNothing(id, "not an actor")
153 }
154 }
155 }
156 }()
157
158 return out, nil
159}
160
161// exportBug publish bugs and related events
162func (ge *githubExporter) exportBug(ctx context.Context, b *cache.BugCache, since time.Time, out chan<- core.ExportResult) {
163 snapshot := b.Snapshot()
164
165 var bugGithubID string
166 var bugGithubURL string
167
168 // Special case:
169 // if a user try to export a bug that is not already exported to Github (or imported
170 // from Github) and we do not have the token of the bug author, there is nothing we can do.
171
172 // first operation is always createOp
173 createOp := snapshot.Operations[0].(*bug.CreateOperation)
174 author := snapshot.Author
175
176 // skip bug if origin is not allowed
177 origin, ok := snapshot.GetCreateMetadata(core.MetaKeyOrigin)
178 if ok && origin != target {
179 out <- core.NewExportNothing(b.Id(), fmt.Sprintf("issue tagged with origin: %s", origin))
180 return
181 }
182
183 // get github bug ID
184 githubID, ok := snapshot.GetCreateMetadata(metaKeyGithubId)
185 if ok {
186 githubURL, ok := snapshot.GetCreateMetadata(metaKeyGithubUrl)
187 if !ok {
188 // if we find github ID, github URL must be found too
189 err := fmt.Errorf("incomplete Github metadata: expected to find issue URL")
190 out <- core.NewExportError(err, b.Id())
191 }
192
193 // extract owner and project
194 owner, project, err := splitURL(githubURL)
195 if err != nil {
196 err := fmt.Errorf("bad project url: %v", err)
197 out <- core.NewExportError(err, b.Id())
198 return
199 }
200
201 // ignore issue comming from other repositories
202 if owner != ge.conf[keyOwner] && project != ge.conf[keyProject] {
203 out <- core.NewExportNothing(b.Id(), fmt.Sprintf("skipping issue from url:%s", githubURL))
204 return
205 }
206
207 out <- core.NewExportNothing(b.Id(), "bug already exported")
208 // will be used to mark operation related to a bug as exported
209 bugGithubID = githubID
210 bugGithubURL = githubURL
211
212 } else {
213 // check that we have a token for operation author
214 client, err := ge.getIdentityClient(author.Id())
215 if err != nil {
216 // if bug is still not exported and we do not have the author stop the execution
217 out <- core.NewExportNothing(b.Id(), fmt.Sprintf("missing author token"))
218 return
219 }
220
221 // create bug
222 id, url, err := createGithubIssue(ctx, client, ge.repositoryID, createOp.Title, createOp.Message)
223 if err != nil {
224 err := errors.Wrap(err, "exporting github issue")
225 out <- core.NewExportError(err, b.Id())
226 return
227 }
228
229 out <- core.NewExportBug(b.Id())
230
231 // mark bug creation operation as exported
232 if err := markOperationAsExported(b, createOp.Id(), id, url); err != nil {
233 err := errors.Wrap(err, "marking operation as exported")
234 out <- core.NewExportError(err, b.Id())
235 return
236 }
237
238 // commit operation to avoid creating multiple issues with multiple pushes
239 if err := b.CommitAsNeeded(); err != nil {
240 err := errors.Wrap(err, "bug commit")
241 out <- core.NewExportError(err, b.Id())
242 return
243 }
244
245 // cache bug github ID and URL
246 bugGithubID = id
247 bugGithubURL = url
248 }
249
250 // cache operation github id
251 ge.cachedOperationIDs[createOp.Id()] = bugGithubID
252
253 for _, op := range snapshot.Operations[1:] {
254 // ignore SetMetadata operations
255 if _, ok := op.(*bug.SetMetadataOperation); ok {
256 continue
257 }
258
259 // ignore operations already existing in github (due to import or export)
260 // cache the ID of already exported or imported issues and events from Github
261 if id, ok := op.GetMetadata(metaKeyGithubId); ok {
262 ge.cachedOperationIDs[op.Id()] = id
263 out <- core.NewExportNothing(op.Id(), "already exported operation")
264 continue
265 }
266
267 opAuthor := op.GetAuthor()
268 client, err := ge.getIdentityClient(opAuthor.Id())
269 if err != nil {
270 out <- core.NewExportNothing(op.Id(), "missing operation author token")
271 continue
272 }
273
274 var id, url string
275 switch op.(type) {
276 case *bug.AddCommentOperation:
277 opr := op.(*bug.AddCommentOperation)
278
279 // send operation to github
280 id, url, err = addCommentGithubIssue(ctx, client, bugGithubID, opr.Message)
281 if err != nil {
282 err := errors.Wrap(err, "adding comment")
283 out <- core.NewExportError(err, b.Id())
284 return
285 }
286
287 out <- core.NewExportComment(op.Id())
288
289 // cache comment id
290 ge.cachedOperationIDs[op.Id()] = id
291
292 case *bug.EditCommentOperation:
293
294 opr := op.(*bug.EditCommentOperation)
295
296 // Since github doesn't consider the issue body as a comment
297 if opr.Target == createOp.Id() {
298
299 // case bug creation operation: we need to edit the Github issue
300 if err := updateGithubIssueBody(ctx, client, bugGithubID, opr.Message); err != nil {
301 err := errors.Wrap(err, "editing issue")
302 out <- core.NewExportError(err, b.Id())
303 return
304 }
305
306 out <- core.NewExportCommentEdition(op.Id())
307
308 id = bugGithubID
309 url = bugGithubURL
310
311 } else {
312
313 // case comment edition operation: we need to edit the Github comment
314 commentID, ok := ge.cachedOperationIDs[opr.Target]
315 if !ok {
316 panic("unexpected error: comment id not found")
317 }
318
319 eid, eurl, err := editCommentGithubIssue(ctx, client, commentID, opr.Message)
320 if err != nil {
321 err := errors.Wrap(err, "editing comment")
322 out <- core.NewExportError(err, b.Id())
323 return
324 }
325
326 out <- core.NewExportCommentEdition(op.Id())
327
328 // use comment id/url instead of issue id/url
329 id = eid
330 url = eurl
331 }
332
333 case *bug.SetStatusOperation:
334 opr := op.(*bug.SetStatusOperation)
335 if err := updateGithubIssueStatus(ctx, client, bugGithubID, opr.Status); err != nil {
336 err := errors.Wrap(err, "editing status")
337 out <- core.NewExportError(err, b.Id())
338 return
339 }
340
341 out <- core.NewExportStatusChange(op.Id())
342
343 id = bugGithubID
344 url = bugGithubURL
345
346 case *bug.SetTitleOperation:
347 opr := op.(*bug.SetTitleOperation)
348 if err := updateGithubIssueTitle(ctx, client, bugGithubID, opr.Title); err != nil {
349 err := errors.Wrap(err, "editing title")
350 out <- core.NewExportError(err, b.Id())
351 return
352 }
353
354 out <- core.NewExportTitleEdition(op.Id())
355
356 id = bugGithubID
357 url = bugGithubURL
358
359 case *bug.LabelChangeOperation:
360 opr := op.(*bug.LabelChangeOperation)
361 if err := ge.updateGithubIssueLabels(ctx, client, bugGithubID, opr.Added, opr.Removed); err != nil {
362 err := errors.Wrap(err, "updating labels")
363 out <- core.NewExportError(err, b.Id())
364 return
365 }
366
367 out <- core.NewExportLabelChange(op.Id())
368
369 id = bugGithubID
370 url = bugGithubURL
371
372 default:
373 panic("unhandled operation type case")
374 }
375
376 // mark operation as exported
377 if err := markOperationAsExported(b, op.Id(), id, url); err != nil {
378 err := errors.Wrap(err, "marking operation as exported")
379 out <- core.NewExportError(err, b.Id())
380 return
381 }
382
383 // commit at each operation export to avoid exporting same events multiple times
384 if err := b.CommitAsNeeded(); err != nil {
385 err := errors.Wrap(err, "bug commit")
386 out <- core.NewExportError(err, b.Id())
387 return
388 }
389 }
390}
391
392// getRepositoryNodeID request github api v3 to get repository node id
393func getRepositoryNodeID(ctx context.Context, owner, project, token string) (string, error) {
394 url := fmt.Sprintf("%s/repos/%s/%s", githubV3Url, owner, project)
395 client := &http.Client{}
396
397 req, err := http.NewRequest("GET", url, nil)
398 if err != nil {
399 return "", err
400 }
401
402 // need the token for private repositories
403 req.Header.Set("Authorization", fmt.Sprintf("token %s", token))
404
405 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
406 defer cancel()
407 req = req.WithContext(ctx)
408
409 resp, err := client.Do(req)
410 if err != nil {
411 return "", err
412 }
413
414 if resp.StatusCode != http.StatusOK {
415 return "", fmt.Errorf("HTTP error %v retrieving repository node id", resp.StatusCode)
416 }
417
418 aux := struct {
419 NodeID string `json:"node_id"`
420 }{}
421
422 data, _ := ioutil.ReadAll(resp.Body)
423 err = resp.Body.Close()
424 if err != nil {
425 return "", err
426 }
427
428 err = json.Unmarshal(data, &aux)
429 if err != nil {
430 return "", err
431 }
432
433 return aux.NodeID, nil
434}
435
436func markOperationAsExported(b *cache.BugCache, target entity.Id, githubID, githubURL string) error {
437 _, err := b.SetMetadata(
438 target,
439 map[string]string{
440 metaKeyGithubId: githubID,
441 metaKeyGithubUrl: githubURL,
442 },
443 )
444
445 return err
446}
447
448func (ge *githubExporter) cacheGithubLabels(ctx context.Context, gc *githubv4.Client) error {
449 variables := map[string]interface{}{
450 "owner": githubv4.String(ge.conf[keyOwner]),
451 "name": githubv4.String(ge.conf[keyProject]),
452 "first": githubv4.Int(10),
453 "after": (*githubv4.String)(nil),
454 }
455
456 q := labelsQuery{}
457
458 hasNextPage := true
459 for hasNextPage {
460 // create a new timeout context at each iteration
461 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
462 defer cancel()
463
464 if err := gc.Query(ctx, &q, variables); err != nil {
465 return err
466 }
467
468 for _, label := range q.Repository.Labels.Nodes {
469 ge.cachedLabels[label.Name] = label.ID
470 }
471
472 hasNextPage = q.Repository.Labels.PageInfo.HasNextPage
473 variables["after"] = q.Repository.Labels.PageInfo.EndCursor
474 }
475
476 return nil
477}
478
479func (ge *githubExporter) getLabelID(gc *githubv4.Client, label string) (string, error) {
480 label = strings.ToLower(label)
481 for cachedLabel, ID := range ge.cachedLabels {
482 if label == strings.ToLower(cachedLabel) {
483 return ID, nil
484 }
485 }
486
487 return "", fmt.Errorf("didn't find label id in cache")
488}
489
490// create a new label and return it github id
491// NOTE: since createLabel mutation is still in preview mode we use github api v3 to create labels
492// see https://developer.github.com/v4/mutation/createlabel/ and https://developer.github.com/v4/previews/#labels-preview
493func (ge *githubExporter) createGithubLabel(ctx context.Context, label, color string) (string, error) {
494 url := fmt.Sprintf("%s/repos/%s/%s/labels", githubV3Url, ge.conf[keyOwner], ge.conf[keyProject])
495 client := &http.Client{}
496
497 params := struct {
498 Name string `json:"name"`
499 Color string `json:"color"`
500 Description string `json:"description"`
501 }{
502 Name: label,
503 Color: color,
504 }
505
506 data, err := json.Marshal(params)
507 if err != nil {
508 return "", err
509 }
510
511 req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
512 if err != nil {
513 return "", err
514 }
515
516 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
517 defer cancel()
518 req = req.WithContext(ctx)
519
520 // need the token for private repositories
521 req.Header.Set("Authorization", fmt.Sprintf("token %s", ge.conf[keyToken]))
522
523 resp, err := client.Do(req)
524 if err != nil {
525 return "", err
526 }
527
528 if resp.StatusCode != http.StatusCreated {
529 return "", fmt.Errorf("error creating label: response status %v", resp.StatusCode)
530 }
531
532 aux := struct {
533 ID int `json:"id"`
534 NodeID string `json:"node_id"`
535 Color string `json:"color"`
536 }{}
537
538 data, _ = ioutil.ReadAll(resp.Body)
539 defer resp.Body.Close()
540
541 err = json.Unmarshal(data, &aux)
542 if err != nil {
543 return "", err
544 }
545
546 return aux.NodeID, nil
547}
548
549/**
550// create github label using api v4
551func (ge *githubExporter) createGithubLabelV4(gc *githubv4.Client, label, labelColor string) (string, error) {
552 m := createLabelMutation{}
553 input := createLabelInput{
554 RepositoryID: ge.repositoryID,
555 Name: githubv4.String(label),
556 Color: githubv4.String(labelColor),
557 }
558
559 parentCtx := context.Background()
560 ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout)
561 defer cancel()
562
563 if err := gc.Mutate(ctx, &m, input, nil); err != nil {
564 return "", err
565 }
566
567 return m.CreateLabel.Label.ID, nil
568}
569*/
570
571func (ge *githubExporter) getOrCreateGithubLabelID(ctx context.Context, gc *githubv4.Client, repositoryID string, label bug.Label) (string, error) {
572 // try to get label id from cache
573 labelID, err := ge.getLabelID(gc, string(label))
574 if err == nil {
575 return labelID, nil
576 }
577
578 // RGBA to hex color
579 rgba := label.Color().RGBA()
580 hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B)
581
582 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
583 defer cancel()
584
585 labelID, err = ge.createGithubLabel(ctx, string(label), hexColor)
586 if err != nil {
587 return "", err
588 }
589
590 return labelID, nil
591}
592
593func (ge *githubExporter) getLabelsIDs(ctx context.Context, gc *githubv4.Client, repositoryID string, labels []bug.Label) ([]githubv4.ID, error) {
594 ids := make([]githubv4.ID, 0, len(labels))
595 var err error
596
597 // check labels ids
598 for _, label := range labels {
599 id, ok := ge.cachedLabels[string(label)]
600 if !ok {
601 // try to query label id
602 id, err = ge.getOrCreateGithubLabelID(ctx, gc, repositoryID, label)
603 if err != nil {
604 return nil, errors.Wrap(err, "get or create github label")
605 }
606
607 // cache label id
608 ge.cachedLabels[string(label)] = id
609 }
610
611 ids = append(ids, githubv4.ID(id))
612 }
613
614 return ids, nil
615}
616
617// create a github issue and return it ID
618func createGithubIssue(ctx context.Context, gc *githubv4.Client, repositoryID, title, body string) (string, string, error) {
619 m := &createIssueMutation{}
620 input := githubv4.CreateIssueInput{
621 RepositoryID: repositoryID,
622 Title: githubv4.String(title),
623 Body: (*githubv4.String)(&body),
624 }
625
626 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
627 defer cancel()
628
629 if err := gc.Mutate(ctx, m, input, nil); err != nil {
630 return "", "", err
631 }
632
633 issue := m.CreateIssue.Issue
634 return issue.ID, issue.URL, nil
635}
636
637// add a comment to an issue and return it ID
638func addCommentGithubIssue(ctx context.Context, gc *githubv4.Client, subjectID string, body string) (string, string, error) {
639 m := &addCommentToIssueMutation{}
640 input := githubv4.AddCommentInput{
641 SubjectID: subjectID,
642 Body: githubv4.String(body),
643 }
644
645 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
646 defer cancel()
647
648 if err := gc.Mutate(ctx, m, input, nil); err != nil {
649 return "", "", err
650 }
651
652 node := m.AddComment.CommentEdge.Node
653 return node.ID, node.URL, nil
654}
655
656func editCommentGithubIssue(ctx context.Context, gc *githubv4.Client, commentID, body string) (string, string, error) {
657 m := &updateIssueCommentMutation{}
658 input := githubv4.UpdateIssueCommentInput{
659 ID: commentID,
660 Body: githubv4.String(body),
661 }
662
663 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
664 defer cancel()
665
666 if err := gc.Mutate(ctx, m, input, nil); err != nil {
667 return "", "", err
668 }
669
670 return commentID, m.UpdateIssueComment.IssueComment.URL, nil
671}
672
673func updateGithubIssueStatus(ctx context.Context, gc *githubv4.Client, id string, status bug.Status) error {
674 m := &updateIssueMutation{}
675
676 // set state
677 var state githubv4.IssueState
678
679 switch status {
680 case bug.OpenStatus:
681 state = githubv4.IssueStateOpen
682 case bug.ClosedStatus:
683 state = githubv4.IssueStateClosed
684 default:
685 panic("unknown bug state")
686 }
687
688 input := githubv4.UpdateIssueInput{
689 ID: id,
690 State: &state,
691 }
692
693 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
694 defer cancel()
695
696 if err := gc.Mutate(ctx, m, input, nil); err != nil {
697 return err
698 }
699
700 return nil
701}
702
703func updateGithubIssueBody(ctx context.Context, gc *githubv4.Client, id string, body string) error {
704 m := &updateIssueMutation{}
705 input := githubv4.UpdateIssueInput{
706 ID: id,
707 Body: (*githubv4.String)(&body),
708 }
709
710 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
711 defer cancel()
712
713 if err := gc.Mutate(ctx, m, input, nil); err != nil {
714 return err
715 }
716
717 return nil
718}
719
720func updateGithubIssueTitle(ctx context.Context, gc *githubv4.Client, id, title string) error {
721 m := &updateIssueMutation{}
722 input := githubv4.UpdateIssueInput{
723 ID: id,
724 Title: (*githubv4.String)(&title),
725 }
726
727 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
728 defer cancel()
729
730 if err := gc.Mutate(ctx, m, input, nil); err != nil {
731 return err
732 }
733
734 return nil
735}
736
737// update github issue labels
738func (ge *githubExporter) updateGithubIssueLabels(ctx context.Context, gc *githubv4.Client, labelableID string, added, removed []bug.Label) error {
739 reqCtx, cancel := context.WithTimeout(ctx, defaultTimeout)
740 defer cancel()
741
742 wg, ctx := errgroup.WithContext(ctx)
743 if len(added) > 0 {
744 wg.Go(func() error {
745 addedIDs, err := ge.getLabelsIDs(ctx, gc, labelableID, added)
746 if err != nil {
747 return err
748 }
749
750 m := &addLabelsToLabelableMutation{}
751 inputAdd := githubv4.AddLabelsToLabelableInput{
752 LabelableID: labelableID,
753 LabelIDs: addedIDs,
754 }
755
756 // add labels
757 if err := gc.Mutate(reqCtx, m, inputAdd, nil); err != nil {
758 return err
759 }
760 return nil
761 })
762 }
763
764 if len(removed) > 0 {
765 wg.Go(func() error {
766 removedIDs, err := ge.getLabelsIDs(ctx, gc, labelableID, removed)
767 if err != nil {
768 return err
769 }
770
771 m2 := &removeLabelsFromLabelableMutation{}
772 inputRemove := githubv4.RemoveLabelsFromLabelableInput{
773 LabelableID: labelableID,
774 LabelIDs: removedIDs,
775 }
776
777 // remove label labels
778 if err := gc.Mutate(reqCtx, m2, inputRemove, nil); err != nil {
779 return err
780 }
781 return nil
782 })
783 }
784
785 return wg.Wait()
786}