1package github
2
3import (
4 "bytes"
5 "context"
6 "encoding/json"
7 "fmt"
8 "io/ioutil"
9 "net/http"
10 "strings"
11 "time"
12
13 "github.com/pkg/errors"
14 "github.com/shurcooL/githubv4"
15 "golang.org/x/sync/errgroup"
16
17 "github.com/MichaelMure/git-bug/bridge/core"
18 "github.com/MichaelMure/git-bug/bridge/core/auth"
19 "github.com/MichaelMure/git-bug/bug"
20 "github.com/MichaelMure/git-bug/cache"
21 "github.com/MichaelMure/git-bug/entity"
22 "github.com/MichaelMure/git-bug/repository"
23)
24
25var (
26 ErrMissingIdentityToken = errors.New("missing identity token")
27)
28
29// githubExporter implement the Exporter interface
30type githubExporter struct {
31 conf core.Configuration
32
33 // cache identities clients
34 identityClient map[entity.Id]*githubv4.Client
35
36 // the client to use for non user-specific queries
37 // should be the client of the default user
38 defaultClient *githubv4.Client
39
40 // the token of the default user
41 defaultToken *auth.Token
42
43 // github repository ID
44 repositoryID string
45
46 // cache identifiers used to speed up exporting operations
47 // cleared for each bug
48 cachedOperationIDs map[entity.Id]string
49
50 // cache labels used to speed up exporting labels events
51 cachedLabels map[string]string
52}
53
54// Init .
55func (ge *githubExporter) Init(repo *cache.RepoCache, conf core.Configuration) error {
56 ge.conf = conf
57 ge.identityClient = make(map[entity.Id]*githubv4.Client)
58 ge.cachedOperationIDs = make(map[entity.Id]string)
59 ge.cachedLabels = make(map[string]string)
60
61 user, err := repo.GetUserIdentity()
62 if err != nil {
63 return err
64 }
65
66 // preload all clients
67 err = ge.cacheAllClient(repo)
68 if err != nil {
69 return err
70 }
71
72 ge.defaultClient, err = ge.getClientForIdentity(user.Id())
73 if err != nil {
74 return err
75 }
76
77 creds, err := auth.List(repo, auth.WithUserId(user.Id()), auth.WithTarget(target), auth.WithKind(auth.KindToken))
78 if err != nil {
79 return err
80 }
81
82 if len(creds) == 0 {
83 return ErrMissingIdentityToken
84 }
85
86 ge.defaultToken = creds[0].(*auth.Token)
87
88 return nil
89}
90
91func (ge *githubExporter) cacheAllClient(repo repository.RepoConfig) error {
92 creds, err := auth.List(repo, auth.WithTarget(target), auth.WithKind(auth.KindToken))
93 if err != nil {
94 return err
95 }
96
97 for _, cred := range creds {
98 if _, ok := ge.identityClient[cred.UserId()]; !ok {
99 client := buildClient(creds[0].(*auth.Token))
100 ge.identityClient[cred.UserId()] = client
101 }
102 }
103
104 return nil
105}
106
107// getClientForIdentity return a githubv4 API client configured with the access token of the given identity.
108func (ge *githubExporter) getClientForIdentity(userId entity.Id) (*githubv4.Client, error) {
109 client, ok := ge.identityClient[userId]
110 if ok {
111 return client, nil
112 }
113
114 return nil, ErrMissingIdentityToken
115}
116
117// ExportAll export all event made by the current user to Github
118func (ge *githubExporter) ExportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ExportResult, error) {
119 out := make(chan core.ExportResult)
120
121 var err error
122 // get repository node id
123 ge.repositoryID, err = getRepositoryNodeID(
124 ctx,
125 ge.defaultToken,
126 ge.conf[keyOwner],
127 ge.conf[keyProject],
128 )
129 if err != nil {
130 return nil, err
131 }
132
133 // query all labels
134 err = ge.cacheGithubLabels(ctx, ge.defaultClient)
135 if err != nil {
136 return nil, err
137 }
138
139 go func() {
140 defer close(out)
141
142 allIdentitiesIds := make([]entity.Id, 0, len(ge.identityClient))
143 for id := range ge.identityClient {
144 allIdentitiesIds = append(allIdentitiesIds, id)
145 }
146
147 allBugsIds := repo.AllBugsIds()
148
149 for _, id := range allBugsIds {
150 b, err := repo.ResolveBug(id)
151 if err != nil {
152 out <- core.NewExportError(errors.Wrap(err, "can't load bug"), id)
153 return
154 }
155
156 select {
157
158 case <-ctx.Done():
159 // stop iterating if context cancel function is called
160 return
161
162 default:
163 snapshot := b.Snapshot()
164
165 // ignore issues created before since date
166 // TODO: compare the Lamport time instead of using the unix time
167 if snapshot.CreatedAt.Before(since) {
168 out <- core.NewExportNothing(b.Id(), "bug created before the since date")
169 continue
170 }
171
172 if snapshot.HasAnyActor(allIdentitiesIds...) {
173 // try to export the bug and it associated events
174 ge.exportBug(ctx, b, since, out)
175 }
176 }
177 }
178 }()
179
180 return out, nil
181}
182
183// exportBug publish bugs and related events
184func (ge *githubExporter) exportBug(ctx context.Context, b *cache.BugCache, since time.Time, out chan<- core.ExportResult) {
185 snapshot := b.Snapshot()
186 var bugUpdated bool
187
188 var bugGithubID string
189 var bugGithubURL string
190
191 // Special case:
192 // if a user try to export a bug that is not already exported to Github (or imported
193 // from Github) and we do not have the token of the bug author, there is nothing we can do.
194
195 // first operation is always createOp
196 createOp := snapshot.Operations[0].(*bug.CreateOperation)
197 author := snapshot.Author
198
199 // skip bug if origin is not allowed
200 origin, ok := snapshot.GetCreateMetadata(core.MetaKeyOrigin)
201 if ok && origin != target {
202 out <- core.NewExportNothing(b.Id(), fmt.Sprintf("issue tagged with origin: %s", origin))
203 return
204 }
205
206 // get github bug ID
207 githubID, ok := snapshot.GetCreateMetadata(metaKeyGithubId)
208 if ok {
209 githubURL, ok := snapshot.GetCreateMetadata(metaKeyGithubUrl)
210 if !ok {
211 // if we find github ID, github URL must be found too
212 err := fmt.Errorf("incomplete Github metadata: expected to find issue URL")
213 out <- core.NewExportError(err, b.Id())
214 }
215
216 // extract owner and project
217 owner, project, err := splitURL(githubURL)
218 if err != nil {
219 err := fmt.Errorf("bad project url: %v", err)
220 out <- core.NewExportError(err, b.Id())
221 return
222 }
223
224 // ignore issue coming from other repositories
225 if owner != ge.conf[keyOwner] && project != ge.conf[keyProject] {
226 out <- core.NewExportNothing(b.Id(), fmt.Sprintf("skipping issue from url:%s", githubURL))
227 return
228 }
229
230 // will be used to mark operation related to a bug as exported
231 bugGithubID = githubID
232 bugGithubURL = githubURL
233
234 } else {
235 // check that we have a token for operation author
236 client, err := ge.getClientForIdentity(author.Id())
237 if err != nil {
238 // if bug is still not exported and we do not have the author stop the execution
239 out <- core.NewExportNothing(b.Id(), fmt.Sprintf("missing author token"))
240 return
241 }
242
243 // create bug
244 id, url, err := createGithubIssue(ctx, client, ge.repositoryID, createOp.Title, createOp.Message)
245 if err != nil {
246 err := errors.Wrap(err, "exporting github issue")
247 out <- core.NewExportError(err, b.Id())
248 return
249 }
250
251 out <- core.NewExportBug(b.Id())
252
253 // mark bug creation operation as exported
254 if err := markOperationAsExported(b, createOp.Id(), id, url); err != nil {
255 err := errors.Wrap(err, "marking operation as exported")
256 out <- core.NewExportError(err, b.Id())
257 return
258 }
259
260 // commit operation to avoid creating multiple issues with multiple pushes
261 if err := b.CommitAsNeeded(); err != nil {
262 err := errors.Wrap(err, "bug commit")
263 out <- core.NewExportError(err, b.Id())
264 return
265 }
266
267 // cache bug github ID and URL
268 bugGithubID = id
269 bugGithubURL = url
270 }
271
272 // cache operation github id
273 ge.cachedOperationIDs[createOp.Id()] = bugGithubID
274
275 for _, op := range snapshot.Operations[1:] {
276 // ignore SetMetadata operations
277 if _, ok := op.(*bug.SetMetadataOperation); ok {
278 continue
279 }
280
281 // ignore operations already existing in github (due to import or export)
282 // cache the ID of already exported or imported issues and events from Github
283 if id, ok := op.GetMetadata(metaKeyGithubId); ok {
284 ge.cachedOperationIDs[op.Id()] = id
285 continue
286 }
287
288 opAuthor := op.GetAuthor()
289 client, err := ge.getClientForIdentity(opAuthor.Id())
290 if err != nil {
291 continue
292 }
293
294 var id, url string
295 switch op := op.(type) {
296 case *bug.AddCommentOperation:
297 // send operation to github
298 id, url, err = addCommentGithubIssue(ctx, client, bugGithubID, op.Message)
299 if err != nil {
300 err := errors.Wrap(err, "adding comment")
301 out <- core.NewExportError(err, b.Id())
302 return
303 }
304
305 out <- core.NewExportComment(op.Id())
306
307 // cache comment id
308 ge.cachedOperationIDs[op.Id()] = id
309
310 case *bug.EditCommentOperation:
311 // Since github doesn't consider the issue body as a comment
312 if op.Target == createOp.Id() {
313
314 // case bug creation operation: we need to edit the Github issue
315 if err := updateGithubIssueBody(ctx, client, bugGithubID, op.Message); err != nil {
316 err := errors.Wrap(err, "editing issue")
317 out <- core.NewExportError(err, b.Id())
318 return
319 }
320
321 out <- core.NewExportCommentEdition(op.Id())
322
323 id = bugGithubID
324 url = bugGithubURL
325
326 } else {
327
328 // case comment edition operation: we need to edit the Github comment
329 commentID, ok := ge.cachedOperationIDs[op.Target]
330 if !ok {
331 panic("unexpected error: comment id not found")
332 }
333
334 eid, eurl, err := editCommentGithubIssue(ctx, client, commentID, op.Message)
335 if err != nil {
336 err := errors.Wrap(err, "editing comment")
337 out <- core.NewExportError(err, b.Id())
338 return
339 }
340
341 out <- core.NewExportCommentEdition(op.Id())
342
343 // use comment id/url instead of issue id/url
344 id = eid
345 url = eurl
346 }
347
348 case *bug.SetStatusOperation:
349 if err := updateGithubIssueStatus(ctx, client, bugGithubID, op.Status); err != nil {
350 err := errors.Wrap(err, "editing status")
351 out <- core.NewExportError(err, b.Id())
352 return
353 }
354
355 out <- core.NewExportStatusChange(op.Id())
356
357 id = bugGithubID
358 url = bugGithubURL
359
360 case *bug.SetTitleOperation:
361 if err := updateGithubIssueTitle(ctx, client, bugGithubID, op.Title); err != nil {
362 err := errors.Wrap(err, "editing title")
363 out <- core.NewExportError(err, b.Id())
364 return
365 }
366
367 out <- core.NewExportTitleEdition(op.Id())
368
369 id = bugGithubID
370 url = bugGithubURL
371
372 case *bug.LabelChangeOperation:
373 if err := ge.updateGithubIssueLabels(ctx, client, bugGithubID, op.Added, op.Removed); err != nil {
374 err := errors.Wrap(err, "updating labels")
375 out <- core.NewExportError(err, b.Id())
376 return
377 }
378
379 out <- core.NewExportLabelChange(op.Id())
380
381 id = bugGithubID
382 url = bugGithubURL
383
384 default:
385 panic("unhandled operation type case")
386 }
387
388 // mark operation as exported
389 if err := markOperationAsExported(b, op.Id(), id, url); err != nil {
390 err := errors.Wrap(err, "marking operation as exported")
391 out <- core.NewExportError(err, b.Id())
392 return
393 }
394
395 // commit at each operation export to avoid exporting same events multiple times
396 if err := b.CommitAsNeeded(); err != nil {
397 err := errors.Wrap(err, "bug commit")
398 out <- core.NewExportError(err, b.Id())
399 return
400 }
401
402 bugUpdated = true
403 }
404
405 if !bugUpdated {
406 out <- core.NewExportNothing(b.Id(), "nothing has been exported")
407 }
408}
409
410// getRepositoryNodeID request github api v3 to get repository node id
411func getRepositoryNodeID(ctx context.Context, token *auth.Token, owner, project string) (string, error) {
412 url := fmt.Sprintf("%s/repos/%s/%s", githubV3Url, owner, project)
413 client := &http.Client{}
414
415 req, err := http.NewRequest("GET", url, nil)
416 if err != nil {
417 return "", err
418 }
419
420 // need the token for private repositories
421 req.Header.Set("Authorization", fmt.Sprintf("token %s", token.Value))
422
423 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
424 defer cancel()
425 req = req.WithContext(ctx)
426
427 resp, err := client.Do(req)
428 if err != nil {
429 return "", err
430 }
431
432 if resp.StatusCode != http.StatusOK {
433 return "", fmt.Errorf("HTTP error %v retrieving repository node id", resp.StatusCode)
434 }
435
436 aux := struct {
437 NodeID string `json:"node_id"`
438 }{}
439
440 data, _ := ioutil.ReadAll(resp.Body)
441 err = resp.Body.Close()
442 if err != nil {
443 return "", err
444 }
445
446 err = json.Unmarshal(data, &aux)
447 if err != nil {
448 return "", err
449 }
450
451 return aux.NodeID, nil
452}
453
454func markOperationAsExported(b *cache.BugCache, target entity.Id, githubID, githubURL string) error {
455 _, err := b.SetMetadata(
456 target,
457 map[string]string{
458 metaKeyGithubId: githubID,
459 metaKeyGithubUrl: githubURL,
460 },
461 )
462
463 return err
464}
465
466func (ge *githubExporter) cacheGithubLabels(ctx context.Context, gc *githubv4.Client) error {
467 variables := map[string]interface{}{
468 "owner": githubv4.String(ge.conf[keyOwner]),
469 "name": githubv4.String(ge.conf[keyProject]),
470 "first": githubv4.Int(10),
471 "after": (*githubv4.String)(nil),
472 }
473
474 q := labelsQuery{}
475
476 hasNextPage := true
477 for hasNextPage {
478 // create a new timeout context at each iteration
479 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
480 defer cancel()
481
482 if err := gc.Query(ctx, &q, variables); err != nil {
483 return err
484 }
485
486 for _, label := range q.Repository.Labels.Nodes {
487 ge.cachedLabels[label.Name] = label.ID
488 }
489
490 hasNextPage = q.Repository.Labels.PageInfo.HasNextPage
491 variables["after"] = q.Repository.Labels.PageInfo.EndCursor
492 }
493
494 return nil
495}
496
497func (ge *githubExporter) getLabelID(gc *githubv4.Client, label string) (string, error) {
498 label = strings.ToLower(label)
499 for cachedLabel, ID := range ge.cachedLabels {
500 if label == strings.ToLower(cachedLabel) {
501 return ID, nil
502 }
503 }
504
505 return "", fmt.Errorf("didn't find label id in cache")
506}
507
508// create a new label and return it github id
509// NOTE: since createLabel mutation is still in preview mode we use github api v3 to create labels
510// see https://developer.github.com/v4/mutation/createlabel/ and https://developer.github.com/v4/previews/#labels-preview
511func (ge *githubExporter) createGithubLabel(ctx context.Context, label, color string) (string, error) {
512 url := fmt.Sprintf("%s/repos/%s/%s/labels", githubV3Url, ge.conf[keyOwner], ge.conf[keyProject])
513 client := &http.Client{}
514
515 params := struct {
516 Name string `json:"name"`
517 Color string `json:"color"`
518 Description string `json:"description"`
519 }{
520 Name: label,
521 Color: color,
522 }
523
524 data, err := json.Marshal(params)
525 if err != nil {
526 return "", err
527 }
528
529 req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
530 if err != nil {
531 return "", err
532 }
533
534 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
535 defer cancel()
536 req = req.WithContext(ctx)
537
538 // need the token for private repositories
539 req.Header.Set("Authorization", fmt.Sprintf("token %s", ge.defaultToken.Value))
540
541 resp, err := client.Do(req)
542 if err != nil {
543 return "", err
544 }
545
546 if resp.StatusCode != http.StatusCreated {
547 return "", fmt.Errorf("error creating label: response status %v", resp.StatusCode)
548 }
549
550 aux := struct {
551 ID int `json:"id"`
552 NodeID string `json:"node_id"`
553 Color string `json:"color"`
554 }{}
555
556 data, _ = ioutil.ReadAll(resp.Body)
557 defer resp.Body.Close()
558
559 err = json.Unmarshal(data, &aux)
560 if err != nil {
561 return "", err
562 }
563
564 return aux.NodeID, nil
565}
566
567/**
568// create github label using api v4
569func (ge *githubExporter) createGithubLabelV4(gc *githubv4.Client, label, labelColor string) (string, error) {
570 m := createLabelMutation{}
571 input := createLabelInput{
572 RepositoryID: ge.repositoryID,
573 Name: githubv4.String(label),
574 Color: githubv4.String(labelColor),
575 }
576
577 parentCtx := context.Background()
578 ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout)
579 defer cancel()
580
581 if err := gc.Mutate(ctx, &m, input, nil); err != nil {
582 return "", err
583 }
584
585 return m.CreateLabel.Label.ID, nil
586}
587*/
588
589func (ge *githubExporter) getOrCreateGithubLabelID(ctx context.Context, gc *githubv4.Client, repositoryID string, label bug.Label) (string, error) {
590 // try to get label id from cache
591 labelID, err := ge.getLabelID(gc, string(label))
592 if err == nil {
593 return labelID, nil
594 }
595
596 // RGBA to hex color
597 rgba := label.Color().RGBA()
598 hexColor := fmt.Sprintf("%.2x%.2x%.2x", rgba.R, rgba.G, rgba.B)
599
600 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
601 defer cancel()
602
603 labelID, err = ge.createGithubLabel(ctx, string(label), hexColor)
604 if err != nil {
605 return "", err
606 }
607
608 return labelID, nil
609}
610
611func (ge *githubExporter) getLabelsIDs(ctx context.Context, gc *githubv4.Client, repositoryID string, labels []bug.Label) ([]githubv4.ID, error) {
612 ids := make([]githubv4.ID, 0, len(labels))
613 var err error
614
615 // check labels ids
616 for _, label := range labels {
617 id, ok := ge.cachedLabels[string(label)]
618 if !ok {
619 // try to query label id
620 id, err = ge.getOrCreateGithubLabelID(ctx, gc, repositoryID, label)
621 if err != nil {
622 return nil, errors.Wrap(err, "get or create github label")
623 }
624
625 // cache label id
626 ge.cachedLabels[string(label)] = id
627 }
628
629 ids = append(ids, githubv4.ID(id))
630 }
631
632 return ids, nil
633}
634
635// create a github issue and return it ID
636func createGithubIssue(ctx context.Context, gc *githubv4.Client, repositoryID, title, body string) (string, string, error) {
637 m := &createIssueMutation{}
638 input := githubv4.CreateIssueInput{
639 RepositoryID: repositoryID,
640 Title: githubv4.String(title),
641 Body: (*githubv4.String)(&body),
642 }
643
644 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
645 defer cancel()
646
647 if err := gc.Mutate(ctx, m, input, nil); err != nil {
648 return "", "", err
649 }
650
651 issue := m.CreateIssue.Issue
652 return issue.ID, issue.URL, nil
653}
654
655// add a comment to an issue and return it ID
656func addCommentGithubIssue(ctx context.Context, gc *githubv4.Client, subjectID string, body string) (string, string, error) {
657 m := &addCommentToIssueMutation{}
658 input := githubv4.AddCommentInput{
659 SubjectID: subjectID,
660 Body: githubv4.String(body),
661 }
662
663 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
664 defer cancel()
665
666 if err := gc.Mutate(ctx, m, input, nil); err != nil {
667 return "", "", err
668 }
669
670 node := m.AddComment.CommentEdge.Node
671 return node.ID, node.URL, nil
672}
673
674func editCommentGithubIssue(ctx context.Context, gc *githubv4.Client, commentID, body string) (string, string, error) {
675 m := &updateIssueCommentMutation{}
676 input := githubv4.UpdateIssueCommentInput{
677 ID: commentID,
678 Body: githubv4.String(body),
679 }
680
681 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
682 defer cancel()
683
684 if err := gc.Mutate(ctx, m, input, nil); err != nil {
685 return "", "", err
686 }
687
688 return commentID, m.UpdateIssueComment.IssueComment.URL, nil
689}
690
691func updateGithubIssueStatus(ctx context.Context, gc *githubv4.Client, id string, status bug.Status) error {
692 m := &updateIssueMutation{}
693
694 // set state
695 var state githubv4.IssueState
696
697 switch status {
698 case bug.OpenStatus:
699 state = githubv4.IssueStateOpen
700 case bug.ClosedStatus:
701 state = githubv4.IssueStateClosed
702 default:
703 panic("unknown bug state")
704 }
705
706 input := githubv4.UpdateIssueInput{
707 ID: id,
708 State: &state,
709 }
710
711 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
712 defer cancel()
713
714 if err := gc.Mutate(ctx, m, input, nil); err != nil {
715 return err
716 }
717
718 return nil
719}
720
721func updateGithubIssueBody(ctx context.Context, gc *githubv4.Client, id string, body string) error {
722 m := &updateIssueMutation{}
723 input := githubv4.UpdateIssueInput{
724 ID: id,
725 Body: (*githubv4.String)(&body),
726 }
727
728 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
729 defer cancel()
730
731 if err := gc.Mutate(ctx, m, input, nil); err != nil {
732 return err
733 }
734
735 return nil
736}
737
738func updateGithubIssueTitle(ctx context.Context, gc *githubv4.Client, id, title string) error {
739 m := &updateIssueMutation{}
740 input := githubv4.UpdateIssueInput{
741 ID: id,
742 Title: (*githubv4.String)(&title),
743 }
744
745 ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
746 defer cancel()
747
748 if err := gc.Mutate(ctx, m, input, nil); err != nil {
749 return err
750 }
751
752 return nil
753}
754
755// update github issue labels
756func (ge *githubExporter) updateGithubIssueLabels(ctx context.Context, gc *githubv4.Client, labelableID string, added, removed []bug.Label) error {
757 reqCtx, cancel := context.WithTimeout(ctx, defaultTimeout)
758 defer cancel()
759
760 wg, ctx := errgroup.WithContext(ctx)
761 if len(added) > 0 {
762 wg.Go(func() error {
763 addedIDs, err := ge.getLabelsIDs(ctx, gc, labelableID, added)
764 if err != nil {
765 return err
766 }
767
768 m := &addLabelsToLabelableMutation{}
769 inputAdd := githubv4.AddLabelsToLabelableInput{
770 LabelableID: labelableID,
771 LabelIDs: addedIDs,
772 }
773
774 // add labels
775 if err := gc.Mutate(reqCtx, m, inputAdd, nil); err != nil {
776 return err
777 }
778 return nil
779 })
780 }
781
782 if len(removed) > 0 {
783 wg.Go(func() error {
784 removedIDs, err := ge.getLabelsIDs(ctx, gc, labelableID, removed)
785 if err != nil {
786 return err
787 }
788
789 m2 := &removeLabelsFromLabelableMutation{}
790 inputRemove := githubv4.RemoveLabelsFromLabelableInput{
791 LabelableID: labelableID,
792 LabelIDs: removedIDs,
793 }
794
795 // remove label labels
796 if err := gc.Mutate(reqCtx, m2, inputRemove, nil); err != nil {
797 return err
798 }
799 return nil
800 })
801 }
802
803 return wg.Wait()
804}