1package github
2
3import (
4 "context"
5 "fmt"
6 "strings"
7
8 "github.com/MichaelMure/git-bug/bridge/core"
9 "github.com/MichaelMure/git-bug/bug"
10 "github.com/MichaelMure/git-bug/cache"
11 "github.com/MichaelMure/git-bug/identity"
12 "github.com/MichaelMure/git-bug/util/git"
13 "github.com/shurcooL/githubv4"
14)
15
16const keyGithubId = "github-id"
17const keyGithubUrl = "github-url"
18const keyGithubLogin = "github-login"
19
20// githubImporter implement the Importer interface
21type githubImporter struct {
22 client *githubv4.Client
23 conf core.Configuration
24}
25
26func (gi *githubImporter) Init(conf core.Configuration) error {
27 gi.conf = conf
28 gi.client = buildClient(conf)
29
30 return nil
31}
32
33func (gi *githubImporter) ImportAll(repo *cache.RepoCache) error {
34 q := &issueTimelineQuery{}
35 variables := map[string]interface{}{
36 "owner": githubv4.String(gi.conf[keyUser]),
37 "name": githubv4.String(gi.conf[keyProject]),
38 "issueFirst": githubv4.Int(1),
39 "issueAfter": (*githubv4.String)(nil),
40 "timelineFirst": githubv4.Int(10),
41 "timelineAfter": (*githubv4.String)(nil),
42
43 // Fun fact, github provide the comment edition in reverse chronological
44 // order, because haha. Look at me, I'm dying of laughter.
45 "issueEditLast": githubv4.Int(10),
46 "issueEditBefore": (*githubv4.String)(nil),
47 "commentEditLast": githubv4.Int(10),
48 "commentEditBefore": (*githubv4.String)(nil),
49 }
50
51 var b *cache.BugCache
52
53 for {
54 err := gi.client.Query(context.TODO(), &q, variables)
55 if err != nil {
56 return err
57 }
58
59 if len(q.Repository.Issues.Nodes) == 0 {
60 return nil
61 }
62
63 issue := q.Repository.Issues.Nodes[0]
64
65 if b == nil {
66 b, err = gi.ensureIssue(repo, issue, variables)
67 if err != nil {
68 return err
69 }
70 }
71
72 for _, itemEdge := range q.Repository.Issues.Nodes[0].Timeline.Edges {
73 err = gi.ensureTimelineItem(repo, b, itemEdge.Cursor, itemEdge.Node, variables)
74 if err != nil {
75 return err
76 }
77 }
78
79 if !issue.Timeline.PageInfo.HasNextPage {
80 err = b.CommitAsNeeded()
81 if err != nil {
82 return err
83 }
84
85 b = nil
86
87 if !q.Repository.Issues.PageInfo.HasNextPage {
88 break
89 }
90
91 variables["issueAfter"] = githubv4.NewString(q.Repository.Issues.PageInfo.EndCursor)
92 variables["timelineAfter"] = (*githubv4.String)(nil)
93 continue
94 }
95
96 variables["timelineAfter"] = githubv4.NewString(issue.Timeline.PageInfo.EndCursor)
97 }
98
99 return nil
100}
101
102func (gi *githubImporter) Import(repo *cache.RepoCache, id string) error {
103 fmt.Println("IMPORT")
104
105 return nil
106}
107
108func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline, rootVariables map[string]interface{}) (*cache.BugCache, error) {
109 fmt.Printf("import issue: %s\n", issue.Title)
110
111 author, err := gi.ensurePerson(repo, issue.Author)
112 if err != nil {
113 return nil, err
114 }
115
116 b, err := repo.ResolveBugCreateMetadata(keyGithubId, parseId(issue.Id))
117 if err != nil && err != bug.ErrBugNotExist {
118 return nil, err
119 }
120
121 // if there is no edit, the UserContentEdits given by github is empty. That
122 // means that the original message is given by the issue message.
123 //
124 // if there is edits, the UserContentEdits given by github contains both the
125 // original message and the following edits. The issue message give the last
126 // version so we don't care about that.
127 //
128 // the tricky part: for an issue older than the UserContentEdits API, github
129 // doesn't have the previous message version anymore and give an edition
130 // with .Diff == nil. We have to filter them.
131
132 if len(issue.UserContentEdits.Nodes) == 0 {
133 if err == bug.ErrBugNotExist {
134 b, err = repo.NewBugRaw(
135 author,
136 issue.CreatedAt.Unix(),
137 // Todo: this might not be the initial title, we need to query the
138 // timeline to be sure
139 issue.Title,
140 cleanupText(string(issue.Body)),
141 nil,
142 map[string]string{
143 keyGithubId: parseId(issue.Id),
144 keyGithubUrl: issue.Url.String(),
145 },
146 )
147 if err != nil {
148 return nil, err
149 }
150 }
151
152 return b, nil
153 }
154
155 // reverse the order, because github
156 reverseEdits(issue.UserContentEdits.Nodes)
157
158 for i, edit := range issue.UserContentEdits.Nodes {
159 if b != nil && i == 0 {
160 // The first edit in the github result is the creation itself, we already have that
161 continue
162 }
163
164 if b == nil {
165 if edit.Diff == nil {
166 // not enough data given by github for old edit, ignore them
167 continue
168 }
169
170 // we create the bug as soon as we have a legit first edition
171 b, err = repo.NewBugRaw(
172 author,
173 issue.CreatedAt.Unix(),
174 // Todo: this might not be the initial title, we need to query the
175 // timeline to be sure
176 issue.Title,
177 cleanupText(string(*edit.Diff)),
178 nil,
179 map[string]string{
180 keyGithubId: parseId(issue.Id),
181 keyGithubUrl: issue.Url.String(),
182 },
183 )
184 if err != nil {
185 return nil, err
186 }
187 continue
188 }
189
190 target, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(issue.Id))
191 if err != nil {
192 return nil, err
193 }
194
195 err = gi.ensureCommentEdit(repo, b, target, edit)
196 if err != nil {
197 return nil, err
198 }
199 }
200
201 if !issue.UserContentEdits.PageInfo.HasNextPage {
202 // if we still didn't get a legit edit, create the bug from the issue data
203 if b == nil {
204 return repo.NewBugRaw(
205 author,
206 issue.CreatedAt.Unix(),
207 // Todo: this might not be the initial title, we need to query the
208 // timeline to be sure
209 issue.Title,
210 cleanupText(string(issue.Body)),
211 nil,
212 map[string]string{
213 keyGithubId: parseId(issue.Id),
214 keyGithubUrl: issue.Url.String(),
215 },
216 )
217 }
218 return b, nil
219 }
220
221 // We have more edit, querying them
222
223 q := &issueEditQuery{}
224 variables := map[string]interface{}{
225 "owner": rootVariables["owner"],
226 "name": rootVariables["name"],
227 "issueFirst": rootVariables["issueFirst"],
228 "issueAfter": rootVariables["issueAfter"],
229 "issueEditLast": githubv4.Int(10),
230 "issueEditBefore": issue.UserContentEdits.PageInfo.StartCursor,
231 }
232
233 for {
234 err := gi.client.Query(context.TODO(), &q, variables)
235 if err != nil {
236 return nil, err
237 }
238
239 edits := q.Repository.Issues.Nodes[0].UserContentEdits
240
241 if len(edits.Nodes) == 0 {
242 return b, nil
243 }
244
245 for _, edit := range edits.Nodes {
246 if b == nil {
247 if edit.Diff == nil {
248 // not enough data given by github for old edit, ignore them
249 continue
250 }
251
252 // we create the bug as soon as we have a legit first edition
253 b, err = repo.NewBugRaw(
254 author,
255 issue.CreatedAt.Unix(),
256 // Todo: this might not be the initial title, we need to query the
257 // timeline to be sure
258 issue.Title,
259 cleanupText(string(*edit.Diff)),
260 nil,
261 map[string]string{
262 keyGithubId: parseId(issue.Id),
263 keyGithubUrl: issue.Url.String(),
264 },
265 )
266 if err != nil {
267 return nil, err
268 }
269 continue
270 }
271
272 target, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(issue.Id))
273 if err != nil {
274 return nil, err
275 }
276
277 err = gi.ensureCommentEdit(repo, b, target, edit)
278 if err != nil {
279 return nil, err
280 }
281 }
282
283 if !edits.PageInfo.HasNextPage {
284 break
285 }
286
287 variables["issueEditBefore"] = edits.PageInfo.StartCursor
288 }
289
290 // TODO: check + import files
291
292 // if we still didn't get a legit edit, create the bug from the issue data
293 if b == nil {
294 return repo.NewBugRaw(
295 author,
296 issue.CreatedAt.Unix(),
297 // Todo: this might not be the initial title, we need to query the
298 // timeline to be sure
299 issue.Title,
300 cleanupText(string(issue.Body)),
301 nil,
302 map[string]string{
303 keyGithubId: parseId(issue.Id),
304 keyGithubUrl: issue.Url.String(),
305 },
306 )
307 }
308
309 return b, nil
310}
311
312func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, cursor githubv4.String, item timelineItem, rootVariables map[string]interface{}) error {
313 fmt.Printf("import %s\n", item.Typename)
314
315 switch item.Typename {
316 case "IssueComment":
317 return gi.ensureComment(repo, b, cursor, item.IssueComment, rootVariables)
318
319 case "LabeledEvent":
320 id := parseId(item.LabeledEvent.Id)
321 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
322 if err != cache.ErrNoMatchingOp {
323 return err
324 }
325 author, err := gi.ensurePerson(repo, item.LabeledEvent.Actor)
326 if err != nil {
327 return err
328 }
329 _, _, err = b.ChangeLabelsRaw(
330 author,
331 item.LabeledEvent.CreatedAt.Unix(),
332 []string{
333 string(item.LabeledEvent.Label.Name),
334 },
335 nil,
336 map[string]string{keyGithubId: id},
337 )
338 return err
339
340 case "UnlabeledEvent":
341 id := parseId(item.UnlabeledEvent.Id)
342 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
343 if err != cache.ErrNoMatchingOp {
344 return err
345 }
346 author, err := gi.ensurePerson(repo, item.UnlabeledEvent.Actor)
347 if err != nil {
348 return err
349 }
350 _, _, err = b.ChangeLabelsRaw(
351 author,
352 item.UnlabeledEvent.CreatedAt.Unix(),
353 nil,
354 []string{
355 string(item.UnlabeledEvent.Label.Name),
356 },
357 map[string]string{keyGithubId: id},
358 )
359 return err
360
361 case "ClosedEvent":
362 id := parseId(item.ClosedEvent.Id)
363 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
364 if err != cache.ErrNoMatchingOp {
365 return err
366 }
367 author, err := gi.ensurePerson(repo, item.ClosedEvent.Actor)
368 if err != nil {
369 return err
370 }
371 _, err = b.CloseRaw(
372 author,
373 item.ClosedEvent.CreatedAt.Unix(),
374 map[string]string{keyGithubId: id},
375 )
376 return err
377
378 case "ReopenedEvent":
379 id := parseId(item.ReopenedEvent.Id)
380 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
381 if err != cache.ErrNoMatchingOp {
382 return err
383 }
384 author, err := gi.ensurePerson(repo, item.ReopenedEvent.Actor)
385 if err != nil {
386 return err
387 }
388 _, err = b.OpenRaw(
389 author,
390 item.ReopenedEvent.CreatedAt.Unix(),
391 map[string]string{keyGithubId: id},
392 )
393 return err
394
395 case "RenamedTitleEvent":
396 id := parseId(item.RenamedTitleEvent.Id)
397 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
398 if err != cache.ErrNoMatchingOp {
399 return err
400 }
401 author, err := gi.ensurePerson(repo, item.RenamedTitleEvent.Actor)
402 if err != nil {
403 return err
404 }
405 _, err = b.SetTitleRaw(
406 author,
407 item.RenamedTitleEvent.CreatedAt.Unix(),
408 string(item.RenamedTitleEvent.CurrentTitle),
409 map[string]string{keyGithubId: id},
410 )
411 return err
412
413 default:
414 fmt.Println("ignore event ", item.Typename)
415 }
416
417 return nil
418}
419
420func (gi *githubImporter) ensureComment(repo *cache.RepoCache, b *cache.BugCache, cursor githubv4.String, comment issueComment, rootVariables map[string]interface{}) error {
421 author, err := gi.ensurePerson(repo, comment.Author)
422 if err != nil {
423 return err
424 }
425
426 var target git.Hash
427 target, err = b.ResolveOperationWithMetadata(keyGithubId, parseId(comment.Id))
428 if err != nil && err != cache.ErrNoMatchingOp {
429 // real error
430 return err
431 }
432
433 // if there is no edit, the UserContentEdits given by github is empty. That
434 // means that the original message is given by the comment message.
435 //
436 // if there is edits, the UserContentEdits given by github contains both the
437 // original message and the following edits. The comment message give the last
438 // version so we don't care about that.
439 //
440 // the tricky part: for a comment older than the UserContentEdits API, github
441 // doesn't have the previous message version anymore and give an edition
442 // with .Diff == nil. We have to filter them.
443
444 if len(comment.UserContentEdits.Nodes) == 0 {
445 if err == cache.ErrNoMatchingOp {
446 op, err := b.AddCommentRaw(
447 author,
448 comment.CreatedAt.Unix(),
449 cleanupText(string(comment.Body)),
450 nil,
451 map[string]string{
452 keyGithubId: parseId(comment.Id),
453 },
454 )
455 if err != nil {
456 return err
457 }
458
459 target, err = op.Hash()
460 if err != nil {
461 return err
462 }
463 }
464
465 return nil
466 }
467
468 // reverse the order, because github
469 reverseEdits(comment.UserContentEdits.Nodes)
470
471 for i, edit := range comment.UserContentEdits.Nodes {
472 if target != "" && i == 0 {
473 // The first edit in the github result is the comment creation itself, we already have that
474 continue
475 }
476
477 if target == "" {
478 if edit.Diff == nil {
479 // not enough data given by github for old edit, ignore them
480 continue
481 }
482
483 op, err := b.AddCommentRaw(
484 author,
485 comment.CreatedAt.Unix(),
486 cleanupText(string(*edit.Diff)),
487 nil,
488 map[string]string{
489 keyGithubId: parseId(comment.Id),
490 keyGithubUrl: comment.Url.String(),
491 },
492 )
493 if err != nil {
494 return err
495 }
496
497 target, err = op.Hash()
498 if err != nil {
499 return err
500 }
501 }
502
503 err := gi.ensureCommentEdit(repo, b, target, edit)
504 if err != nil {
505 return err
506 }
507 }
508
509 if !comment.UserContentEdits.PageInfo.HasNextPage {
510 return nil
511 }
512
513 // We have more edit, querying them
514
515 q := &commentEditQuery{}
516 variables := map[string]interface{}{
517 "owner": rootVariables["owner"],
518 "name": rootVariables["name"],
519 "issueFirst": rootVariables["issueFirst"],
520 "issueAfter": rootVariables["issueAfter"],
521 "timelineFirst": githubv4.Int(1),
522 "timelineAfter": cursor,
523 "commentEditLast": githubv4.Int(10),
524 "commentEditBefore": comment.UserContentEdits.PageInfo.StartCursor,
525 }
526
527 for {
528 err := gi.client.Query(context.TODO(), &q, variables)
529 if err != nil {
530 return err
531 }
532
533 edits := q.Repository.Issues.Nodes[0].Timeline.Nodes[0].IssueComment.UserContentEdits
534
535 if len(edits.Nodes) == 0 {
536 return nil
537 }
538
539 for i, edit := range edits.Nodes {
540 if i == 0 {
541 // The first edit in the github result is the creation itself, we already have that
542 continue
543 }
544
545 err := gi.ensureCommentEdit(repo, b, target, edit)
546 if err != nil {
547 return err
548 }
549 }
550
551 if !edits.PageInfo.HasNextPage {
552 break
553 }
554
555 variables["commentEditBefore"] = edits.PageInfo.StartCursor
556 }
557
558 // TODO: check + import files
559
560 return nil
561}
562
563func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugCache, target git.Hash, edit userContentEdit) error {
564 if edit.Diff == nil {
565 // this happen if the event is older than early 2018, Github doesn't have the data before that.
566 // Best we can do is to ignore the event.
567 return nil
568 }
569
570 _, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(edit.Id))
571 if err == nil {
572 // already imported
573 return nil
574 }
575 if err != cache.ErrNoMatchingOp {
576 // real error
577 return err
578 }
579
580 fmt.Println("import edition")
581
582 editor, err := gi.ensurePerson(repo, edit.Editor)
583 if err != nil {
584 return err
585 }
586
587 switch {
588 case edit.DeletedAt != nil:
589 // comment deletion, not supported yet
590
591 case edit.DeletedAt == nil:
592 // comment edition
593 _, err := b.EditCommentRaw(
594 editor,
595 edit.CreatedAt.Unix(),
596 target,
597 cleanupText(string(*edit.Diff)),
598 map[string]string{
599 keyGithubId: parseId(edit.Id),
600 },
601 )
602 if err != nil {
603 return err
604 }
605 }
606
607 return nil
608}
609
610// ensurePerson create a bug.Person from the Github data
611func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
612 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
613 // in it's UI. So we need a special case to get it.
614 if actor == nil {
615 return gi.getGhost(repo)
616 }
617
618 // Look first in the cache
619 i, err := repo.ResolveIdentityImmutableMetadata(keyGithubLogin, string(actor.Login))
620 if err == nil {
621 return i, nil
622 }
623 if _, ok := err.(identity.ErrMultipleMatch); ok {
624 return nil, err
625 }
626
627 var name string
628 var email string
629
630 switch actor.Typename {
631 case "User":
632 if actor.User.Name != nil {
633 name = string(*(actor.User.Name))
634 }
635 email = string(actor.User.Email)
636 case "Organization":
637 if actor.Organization.Name != nil {
638 name = string(*(actor.Organization.Name))
639 }
640 if actor.Organization.Email != nil {
641 email = string(*(actor.Organization.Email))
642 }
643 case "Bot":
644 }
645
646 return repo.NewIdentityRaw(
647 name,
648 email,
649 string(actor.Login),
650 string(actor.AvatarUrl),
651 map[string]string{
652 keyGithubLogin: string(actor.Login),
653 },
654 )
655}
656
657func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, error) {
658 // Look first in the cache
659 i, err := repo.ResolveIdentityImmutableMetadata(keyGithubLogin, "ghost")
660 if err == nil {
661 return i, nil
662 }
663 if _, ok := err.(identity.ErrMultipleMatch); ok {
664 return nil, err
665 }
666
667 var q userQuery
668
669 variables := map[string]interface{}{
670 "login": githubv4.String("ghost"),
671 }
672
673 err = gi.client.Query(context.TODO(), &q, variables)
674 if err != nil {
675 return nil, err
676 }
677
678 var name string
679 if q.User.Name != nil {
680 name = string(*q.User.Name)
681 }
682
683 return repo.NewIdentityRaw(
684 name,
685 string(q.User.Email),
686 string(q.User.Login),
687 string(q.User.AvatarUrl),
688 map[string]string{
689 keyGithubLogin: string(q.User.Login),
690 },
691 )
692}
693
694// parseId convert the unusable githubv4.ID (an interface{}) into a string
695func parseId(id githubv4.ID) string {
696 return fmt.Sprintf("%v", id)
697}
698
699func cleanupText(text string) string {
700 // windows new line, Github, really ?
701 text = strings.Replace(text, "\r\n", "\n", -1)
702
703 // trim extra new line not displayed in the github UI but still present in the data
704 return strings.TrimSpace(text)
705}
706
707func reverseEdits(edits []userContentEdit) []userContentEdit {
708 for i, j := 0, len(edits)-1; i < j; i, j = i+1, j-1 {
709 edits[i], edits[j] = edits[j], edits[i]
710 }
711 return edits
712}