1package github
2
3import (
4 "context"
5 "fmt"
6 "strings"
7
8 "github.com/MichaelMure/git-bug/bridge/core"
9 "github.com/MichaelMure/git-bug/bug"
10 "github.com/MichaelMure/git-bug/cache"
11 "github.com/MichaelMure/git-bug/identity"
12 "github.com/MichaelMure/git-bug/util/git"
13 "github.com/shurcooL/githubv4"
14)
15
16const keyGithubId = "github-id"
17const keyGithubUrl = "github-url"
18const keyGithubLogin = "github-login"
19
20// githubImporter implement the Importer interface
21type githubImporter struct {
22 client *githubv4.Client
23 conf core.Configuration
24}
25
26func (gi *githubImporter) Init(conf core.Configuration) error {
27 gi.conf = conf
28 gi.client = buildClient(conf)
29
30 return nil
31}
32
33func (gi *githubImporter) ImportAll(repo *cache.RepoCache) error {
34 q := &issueTimelineQuery{}
35 variables := map[string]interface{}{
36 "owner": githubv4.String(gi.conf[keyUser]),
37 "name": githubv4.String(gi.conf[keyProject]),
38 "issueFirst": githubv4.Int(1),
39 "issueAfter": (*githubv4.String)(nil),
40 "timelineFirst": githubv4.Int(10),
41 "timelineAfter": (*githubv4.String)(nil),
42
43 // Fun fact, github provide the comment edition in reverse chronological
44 // order, because haha. Look at me, I'm dying of laughter.
45 "issueEditLast": githubv4.Int(10),
46 "issueEditBefore": (*githubv4.String)(nil),
47 "commentEditLast": githubv4.Int(10),
48 "commentEditBefore": (*githubv4.String)(nil),
49 }
50
51 var b *cache.BugCache
52
53 for {
54 err := gi.client.Query(context.TODO(), &q, variables)
55 if err != nil {
56 return err
57 }
58
59 if len(q.Repository.Issues.Nodes) == 0 {
60 return nil
61 }
62
63 issue := q.Repository.Issues.Nodes[0]
64
65 if b == nil {
66 b, err = gi.ensureIssue(repo, issue, variables)
67 if err != nil {
68 return err
69 }
70 }
71
72 for _, itemEdge := range q.Repository.Issues.Nodes[0].Timeline.Edges {
73 err = gi.ensureTimelineItem(repo, b, itemEdge.Cursor, itemEdge.Node, variables)
74 if err != nil {
75 return err
76 }
77 }
78
79 if !issue.Timeline.PageInfo.HasNextPage {
80 err = b.CommitAsNeeded()
81 if err != nil {
82 return err
83 }
84
85 b = nil
86
87 if !q.Repository.Issues.PageInfo.HasNextPage {
88 break
89 }
90
91 variables["issueAfter"] = githubv4.NewString(q.Repository.Issues.PageInfo.EndCursor)
92 variables["timelineAfter"] = (*githubv4.String)(nil)
93 continue
94 }
95
96 variables["timelineAfter"] = githubv4.NewString(issue.Timeline.PageInfo.EndCursor)
97 }
98
99 return nil
100}
101
102func (gi *githubImporter) Import(repo *cache.RepoCache, id string) error {
103 fmt.Println("IMPORT")
104
105 return nil
106}
107
108func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline, rootVariables map[string]interface{}) (*cache.BugCache, error) {
109 fmt.Printf("import issue: %s\n", issue.Title)
110
111 author, err := gi.ensurePerson(repo, issue.Author)
112 if err != nil {
113 return nil, err
114 }
115
116 b, err := repo.ResolveBugCreateMetadata(keyGithubId, parseId(issue.Id))
117 if err != nil && err != bug.ErrBugNotExist {
118 return nil, err
119 }
120
121 // if there is no edit, the UserContentEdits given by github is empty. That
122 // means that the original message is given by the issue message.
123 //
124 // if there is edits, the UserContentEdits given by github contains both the
125 // original message and the following edits. The issue message give the last
126 // version so we don't care about that.
127 //
128 // the tricky part: for an issue older than the UserContentEdits API, github
129 // doesn't have the previous message version anymore and give an edition
130 // with .Diff == nil. We have to filter them.
131
132 if len(issue.UserContentEdits.Nodes) == 0 {
133 if err == bug.ErrBugNotExist {
134 b, err = repo.NewBugRaw(
135 author,
136 issue.CreatedAt.Unix(),
137 // Todo: this might not be the initial title, we need to query the
138 // timeline to be sure
139 issue.Title,
140 cleanupText(string(issue.Body)),
141 nil,
142 map[string]string{
143 keyGithubId: parseId(issue.Id),
144 keyGithubUrl: issue.Url.String(),
145 },
146 )
147 if err != nil {
148 return nil, err
149 }
150 }
151
152 return b, nil
153 }
154
155 // reverse the order, because github
156 reverseEdits(issue.UserContentEdits.Nodes)
157
158 for i, edit := range issue.UserContentEdits.Nodes {
159 if b != nil && i == 0 {
160 // The first edit in the github result is the creation itself, we already have that
161 continue
162 }
163
164 if b == nil {
165 if edit.Diff == nil {
166 // not enough data given by github for old edit, ignore them
167 continue
168 }
169
170 // we create the bug as soon as we have a legit first edition
171 b, err = repo.NewBugRaw(
172 author,
173 issue.CreatedAt.Unix(),
174 // Todo: this might not be the initial title, we need to query the
175 // timeline to be sure
176 issue.Title,
177 cleanupText(string(*edit.Diff)),
178 nil,
179 map[string]string{
180 keyGithubId: parseId(issue.Id),
181 keyGithubUrl: issue.Url.String(),
182 },
183 )
184 if err != nil {
185 return nil, err
186 }
187 continue
188 }
189
190 target, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(issue.Id))
191 if err != nil {
192 return nil, err
193 }
194
195 err = gi.ensureCommentEdit(repo, b, target, edit)
196 if err != nil {
197 return nil, err
198 }
199 }
200
201 if !issue.UserContentEdits.PageInfo.HasNextPage {
202 // if we still didn't get a legit edit, create the bug from the issue data
203 if b == nil {
204 return repo.NewBugRaw(
205 author,
206 issue.CreatedAt.Unix(),
207 // Todo: this might not be the initial title, we need to query the
208 // timeline to be sure
209 issue.Title,
210 cleanupText(string(issue.Body)),
211 nil,
212 map[string]string{
213 keyGithubId: parseId(issue.Id),
214 keyGithubUrl: issue.Url.String(),
215 },
216 )
217 }
218 return b, nil
219 }
220
221 // We have more edit, querying them
222
223 q := &issueEditQuery{}
224 variables := map[string]interface{}{
225 "owner": rootVariables["owner"],
226 "name": rootVariables["name"],
227 "issueFirst": rootVariables["issueFirst"],
228 "issueAfter": rootVariables["issueAfter"],
229 "issueEditLast": githubv4.Int(10),
230 "issueEditBefore": issue.UserContentEdits.PageInfo.StartCursor,
231 }
232
233 for {
234 err := gi.client.Query(context.TODO(), &q, variables)
235 if err != nil {
236 return nil, err
237 }
238
239 edits := q.Repository.Issues.Nodes[0].UserContentEdits
240
241 if len(edits.Nodes) == 0 {
242 return b, nil
243 }
244
245 for _, edit := range edits.Nodes {
246 if b == nil {
247 if edit.Diff == nil {
248 // not enough data given by github for old edit, ignore them
249 continue
250 }
251
252 // we create the bug as soon as we have a legit first edition
253 b, err = repo.NewBugRaw(
254 author,
255 issue.CreatedAt.Unix(),
256 // Todo: this might not be the initial title, we need to query the
257 // timeline to be sure
258 issue.Title,
259 cleanupText(string(*edit.Diff)),
260 nil,
261 map[string]string{
262 keyGithubId: parseId(issue.Id),
263 keyGithubUrl: issue.Url.String(),
264 },
265 )
266 if err != nil {
267 return nil, err
268 }
269 continue
270 }
271
272 target, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(issue.Id))
273 if err != nil {
274 return nil, err
275 }
276
277 err = gi.ensureCommentEdit(repo, b, target, edit)
278 if err != nil {
279 return nil, err
280 }
281 }
282
283 if !edits.PageInfo.HasNextPage {
284 break
285 }
286
287 variables["issueEditBefore"] = edits.PageInfo.StartCursor
288 }
289
290 // TODO: check + import files
291
292 // if we still didn't get a legit edit, create the bug from the issue data
293 if b == nil {
294 return repo.NewBugRaw(
295 author,
296 issue.CreatedAt.Unix(),
297 // Todo: this might not be the initial title, we need to query the
298 // timeline to be sure
299 issue.Title,
300 cleanupText(string(issue.Body)),
301 nil,
302 map[string]string{
303 keyGithubId: parseId(issue.Id),
304 keyGithubUrl: issue.Url.String(),
305 },
306 )
307 }
308
309 return b, nil
310}
311
312func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, cursor githubv4.String, item timelineItem, rootVariables map[string]interface{}) error {
313 fmt.Printf("import %s\n", item.Typename)
314
315 switch item.Typename {
316 case "IssueComment":
317 return gi.ensureComment(repo, b, cursor, item.IssueComment, rootVariables)
318
319 case "LabeledEvent":
320 id := parseId(item.LabeledEvent.Id)
321 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
322 if err != cache.ErrNoMatchingOp {
323 return err
324 }
325 author, err := gi.ensurePerson(repo, item.LabeledEvent.Actor)
326 if err != nil {
327 return err
328 }
329 _, _, err = b.ChangeLabelsRaw(
330 author,
331 item.LabeledEvent.CreatedAt.Unix(),
332 []string{
333 string(item.LabeledEvent.Label.Name),
334 },
335 nil,
336 map[string]string{keyGithubId: id},
337 )
338 return err
339
340 case "UnlabeledEvent":
341 id := parseId(item.UnlabeledEvent.Id)
342 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
343 if err != cache.ErrNoMatchingOp {
344 return err
345 }
346 author, err := gi.ensurePerson(repo, item.UnlabeledEvent.Actor)
347 if err != nil {
348 return err
349 }
350 _, _, err = b.ChangeLabelsRaw(
351 author,
352 item.UnlabeledEvent.CreatedAt.Unix(),
353 nil,
354 []string{
355 string(item.UnlabeledEvent.Label.Name),
356 },
357 map[string]string{keyGithubId: id},
358 )
359 return err
360
361 case "ClosedEvent":
362 id := parseId(item.ClosedEvent.Id)
363 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
364 if err != cache.ErrNoMatchingOp {
365 return err
366 }
367 author, err := gi.ensurePerson(repo, item.ClosedEvent.Actor)
368 if err != nil {
369 return err
370 }
371 _, err = b.CloseRaw(
372 author,
373 item.ClosedEvent.CreatedAt.Unix(),
374 map[string]string{keyGithubId: id},
375 )
376 return err
377
378 case "ReopenedEvent":
379 id := parseId(item.ReopenedEvent.Id)
380 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
381 if err != cache.ErrNoMatchingOp {
382 return err
383 }
384 author, err := gi.ensurePerson(repo, item.ReopenedEvent.Actor)
385 if err != nil {
386 return err
387 }
388 _, err = b.OpenRaw(
389 author,
390 item.ReopenedEvent.CreatedAt.Unix(),
391 map[string]string{keyGithubId: id},
392 )
393 return err
394
395 case "RenamedTitleEvent":
396 id := parseId(item.RenamedTitleEvent.Id)
397 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
398 if err != cache.ErrNoMatchingOp {
399 return err
400 }
401 author, err := gi.ensurePerson(repo, item.RenamedTitleEvent.Actor)
402 if err != nil {
403 return err
404 }
405 _, err = b.SetTitleRaw(
406 author,
407 item.RenamedTitleEvent.CreatedAt.Unix(),
408 string(item.RenamedTitleEvent.CurrentTitle),
409 map[string]string{keyGithubId: id},
410 )
411 return err
412
413 default:
414 fmt.Println("ignore event ", item.Typename)
415 }
416
417 return nil
418}
419
420func (gi *githubImporter) ensureComment(repo *cache.RepoCache, b *cache.BugCache, cursor githubv4.String, comment issueComment, rootVariables map[string]interface{}) error {
421 author, err := gi.ensurePerson(repo, comment.Author)
422 if err != nil {
423 return err
424 }
425
426 target, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(comment.Id))
427 if err != nil && err != cache.ErrNoMatchingOp {
428 // real error
429 return err
430 }
431
432 // if there is no edit, the UserContentEdits given by github is empty. That
433 // means that the original message is given by the comment message.
434 //
435 // if there is edits, the UserContentEdits given by github contains both the
436 // original message and the following edits. The comment message give the last
437 // version so we don't care about that.
438 //
439 // the tricky part: for a comment older than the UserContentEdits API, github
440 // doesn't have the previous message version anymore and give an edition
441 // with .Diff == nil. We have to filter them.
442
443 if len(comment.UserContentEdits.Nodes) == 0 {
444 if err == cache.ErrNoMatchingOp {
445 op, err := b.AddCommentRaw(
446 author,
447 comment.CreatedAt.Unix(),
448 cleanupText(string(comment.Body)),
449 nil,
450 map[string]string{
451 keyGithubId: parseId(comment.Id),
452 },
453 )
454 if err != nil {
455 return err
456 }
457
458 target, err = op.Hash()
459 if err != nil {
460 return err
461 }
462 }
463
464 return nil
465 }
466
467 // reverse the order, because github
468 reverseEdits(comment.UserContentEdits.Nodes)
469
470 for i, edit := range comment.UserContentEdits.Nodes {
471 if target != "" && i == 0 {
472 // The first edit in the github result is the comment creation itself, we already have that
473 continue
474 }
475
476 if target == "" {
477 if edit.Diff == nil {
478 // not enough data given by github for old edit, ignore them
479 continue
480 }
481
482 op, err := b.AddCommentRaw(
483 author,
484 comment.CreatedAt.Unix(),
485 cleanupText(string(*edit.Diff)),
486 nil,
487 map[string]string{
488 keyGithubId: parseId(comment.Id),
489 keyGithubUrl: comment.Url.String(),
490 },
491 )
492 if err != nil {
493 return err
494 }
495
496 target, err = op.Hash()
497 if err != nil {
498 return err
499 }
500 }
501
502 err := gi.ensureCommentEdit(repo, b, target, edit)
503 if err != nil {
504 return err
505 }
506 }
507
508 if !comment.UserContentEdits.PageInfo.HasNextPage {
509 return nil
510 }
511
512 // We have more edit, querying them
513
514 q := &commentEditQuery{}
515 variables := map[string]interface{}{
516 "owner": rootVariables["owner"],
517 "name": rootVariables["name"],
518 "issueFirst": rootVariables["issueFirst"],
519 "issueAfter": rootVariables["issueAfter"],
520 "timelineFirst": githubv4.Int(1),
521 "timelineAfter": cursor,
522 "commentEditLast": githubv4.Int(10),
523 "commentEditBefore": comment.UserContentEdits.PageInfo.StartCursor,
524 }
525
526 for {
527 err := gi.client.Query(context.TODO(), &q, variables)
528 if err != nil {
529 return err
530 }
531
532 edits := q.Repository.Issues.Nodes[0].Timeline.Nodes[0].IssueComment.UserContentEdits
533
534 if len(edits.Nodes) == 0 {
535 return nil
536 }
537
538 for i, edit := range edits.Nodes {
539 if i == 0 {
540 // The first edit in the github result is the creation itself, we already have that
541 continue
542 }
543
544 err := gi.ensureCommentEdit(repo, b, target, edit)
545 if err != nil {
546 return err
547 }
548 }
549
550 if !edits.PageInfo.HasNextPage {
551 break
552 }
553
554 variables["commentEditBefore"] = edits.PageInfo.StartCursor
555 }
556
557 // TODO: check + import files
558
559 return nil
560}
561
562func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugCache, target git.Hash, edit userContentEdit) error {
563 if edit.Diff == nil {
564 // this happen if the event is older than early 2018, Github doesn't have the data before that.
565 // Best we can do is to ignore the event.
566 return nil
567 }
568
569 _, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(edit.Id))
570 if err == nil {
571 // already imported
572 return nil
573 }
574 if err != cache.ErrNoMatchingOp {
575 // real error
576 return err
577 }
578
579 fmt.Println("import edition")
580
581 editor, err := gi.ensurePerson(repo, edit.Editor)
582 if err != nil {
583 return err
584 }
585
586 switch {
587 case edit.DeletedAt != nil:
588 // comment deletion, not supported yet
589
590 case edit.DeletedAt == nil:
591 // comment edition
592 _, err := b.EditCommentRaw(
593 editor,
594 edit.CreatedAt.Unix(),
595 target,
596 cleanupText(string(*edit.Diff)),
597 map[string]string{
598 keyGithubId: parseId(edit.Id),
599 },
600 )
601 if err != nil {
602 return err
603 }
604 }
605
606 return nil
607}
608
609// ensurePerson create a bug.Person from the Github data
610func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
611 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
612 // in it's UI. So we need a special case to get it.
613 if actor == nil {
614 return gi.getGhost(repo)
615 }
616
617 // Look first in the cache
618 i, err := repo.ResolveIdentityImmutableMetadata(keyGithubLogin, string(actor.Login))
619 if err == nil {
620 return i, nil
621 }
622 if _, ok := err.(identity.ErrMultipleMatch); ok {
623 return nil, err
624 }
625
626 var name string
627 var email string
628
629 switch actor.Typename {
630 case "User":
631 if actor.User.Name != nil {
632 name = string(*(actor.User.Name))
633 }
634 email = string(actor.User.Email)
635 case "Organization":
636 if actor.Organization.Name != nil {
637 name = string(*(actor.Organization.Name))
638 }
639 if actor.Organization.Email != nil {
640 email = string(*(actor.Organization.Email))
641 }
642 case "Bot":
643 }
644
645 return repo.NewIdentityRaw(
646 name,
647 email,
648 string(actor.Login),
649 string(actor.AvatarUrl),
650 map[string]string{
651 keyGithubLogin: string(actor.Login),
652 },
653 )
654}
655
656func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, error) {
657 // Look first in the cache
658 i, err := repo.ResolveIdentityImmutableMetadata(keyGithubLogin, "ghost")
659 if err == nil {
660 return i, nil
661 }
662 if _, ok := err.(identity.ErrMultipleMatch); ok {
663 return nil, err
664 }
665
666 var q userQuery
667
668 variables := map[string]interface{}{
669 "login": githubv4.String("ghost"),
670 }
671
672 err = gi.client.Query(context.TODO(), &q, variables)
673 if err != nil {
674 return nil, err
675 }
676
677 var name string
678 if q.User.Name != nil {
679 name = string(*q.User.Name)
680 }
681
682 return repo.NewIdentityRaw(
683 name,
684 string(q.User.Email),
685 string(q.User.Login),
686 string(q.User.AvatarUrl),
687 map[string]string{
688 keyGithubLogin: string(q.User.Login),
689 },
690 )
691}
692
693// parseId convert the unusable githubv4.ID (an interface{}) into a string
694func parseId(id githubv4.ID) string {
695 return fmt.Sprintf("%v", id)
696}
697
698func cleanupText(text string) string {
699 // windows new line, Github, really ?
700 text = strings.Replace(text, "\r\n", "\n", -1)
701
702 // trim extra new line not displayed in the github UI but still present in the data
703 return strings.TrimSpace(text)
704}
705
706func reverseEdits(edits []userContentEdit) []userContentEdit {
707 for i, j := 0, len(edits)-1; i < j; i, j = i+1, j-1 {
708 edits[i], edits[j] = edits[j], edits[i]
709 }
710 return edits
711}