1package github
2
3import (
4 "context"
5 "fmt"
6 "strings"
7
8 "github.com/MichaelMure/git-bug/bridge/core"
9 "github.com/MichaelMure/git-bug/bug"
10 "github.com/MichaelMure/git-bug/cache"
11 "github.com/MichaelMure/git-bug/identity"
12 "github.com/MichaelMure/git-bug/util/git"
13 "github.com/shurcooL/githubv4"
14)
15
16const keyGithubId = "github-id"
17const keyGithubUrl = "github-url"
18const keyGithubLogin = "github-login"
19
20// githubImporter implement the Importer interface
21type githubImporter struct {
22 client *githubv4.Client
23 conf core.Configuration
24 ghost identity.Interface
25}
26
27func (gi *githubImporter) Init(conf core.Configuration) error {
28 gi.conf = conf
29 gi.client = buildClient(conf)
30
31 return gi.fetchGhost()
32}
33
34func (gi *githubImporter) ImportAll(repo *cache.RepoCache) error {
35 q := &issueTimelineQuery{}
36 variables := map[string]interface{}{
37 "owner": githubv4.String(gi.conf[keyUser]),
38 "name": githubv4.String(gi.conf[keyProject]),
39 "issueFirst": githubv4.Int(1),
40 "issueAfter": (*githubv4.String)(nil),
41 "timelineFirst": githubv4.Int(10),
42 "timelineAfter": (*githubv4.String)(nil),
43
44 // Fun fact, github provide the comment edition in reverse chronological
45 // order, because haha. Look at me, I'm dying of laughter.
46 "issueEditLast": githubv4.Int(10),
47 "issueEditBefore": (*githubv4.String)(nil),
48 "commentEditLast": githubv4.Int(10),
49 "commentEditBefore": (*githubv4.String)(nil),
50 }
51
52 var b *cache.BugCache
53
54 for {
55 err := gi.client.Query(context.TODO(), &q, variables)
56 if err != nil {
57 return err
58 }
59
60 if len(q.Repository.Issues.Nodes) == 0 {
61 return nil
62 }
63
64 issue := q.Repository.Issues.Nodes[0]
65
66 if b == nil {
67 b, err = gi.ensureIssue(repo, issue, variables)
68 if err != nil {
69 return err
70 }
71 }
72
73 for _, itemEdge := range q.Repository.Issues.Nodes[0].Timeline.Edges {
74 err = gi.ensureTimelineItem(b, itemEdge.Cursor, itemEdge.Node, variables)
75 if err != nil {
76 return err
77 }
78 }
79
80 if !issue.Timeline.PageInfo.HasNextPage {
81 err = b.CommitAsNeeded()
82 if err != nil {
83 return err
84 }
85
86 b = nil
87
88 if !q.Repository.Issues.PageInfo.HasNextPage {
89 break
90 }
91
92 variables["issueAfter"] = githubv4.NewString(q.Repository.Issues.PageInfo.EndCursor)
93 variables["timelineAfter"] = (*githubv4.String)(nil)
94 continue
95 }
96
97 variables["timelineAfter"] = githubv4.NewString(issue.Timeline.PageInfo.EndCursor)
98 }
99
100 return nil
101}
102
103func (gi *githubImporter) Import(repo *cache.RepoCache, id string) error {
104 fmt.Println("IMPORT")
105
106 return nil
107}
108
109func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline, rootVariables map[string]interface{}) (*cache.BugCache, error) {
110 fmt.Printf("import issue: %s\n", issue.Title)
111
112 b, err := repo.ResolveBugCreateMetadata(keyGithubId, parseId(issue.Id))
113 if err != nil && err != bug.ErrBugNotExist {
114 return nil, err
115 }
116
117 // if there is no edit, the UserContentEdits given by github is empty. That
118 // means that the original message is given by the issue message.
119 //
120 // if there is edits, the UserContentEdits given by github contains both the
121 // original message and the following edits. The issue message give the last
122 // version so we don't care about that.
123 //
124 // the tricky part: for an issue older than the UserContentEdits API, github
125 // doesn't have the previous message version anymore and give an edition
126 // with .Diff == nil. We have to filter them.
127
128 if len(issue.UserContentEdits.Nodes) == 0 {
129 if err == bug.ErrBugNotExist {
130 b, err = repo.NewBugRaw(
131 gi.makePerson(issue.Author),
132 issue.CreatedAt.Unix(),
133 // Todo: this might not be the initial title, we need to query the
134 // timeline to be sure
135 issue.Title,
136 cleanupText(string(issue.Body)),
137 nil,
138 map[string]string{
139 keyGithubId: parseId(issue.Id),
140 keyGithubUrl: issue.Url.String(),
141 },
142 )
143
144 if err != nil {
145 return nil, err
146 }
147 }
148
149 return b, nil
150 }
151
152 // reverse the order, because github
153 reverseEdits(issue.UserContentEdits.Nodes)
154
155 for i, edit := range issue.UserContentEdits.Nodes {
156 if b != nil && i == 0 {
157 // The first edit in the github result is the creation itself, we already have that
158 continue
159 }
160
161 if b == nil {
162 if edit.Diff == nil {
163 // not enough data given by github for old edit, ignore them
164 continue
165 }
166
167 // we create the bug as soon as we have a legit first edition
168 b, err = repo.NewBugRaw(
169 gi.makePerson(issue.Author),
170 issue.CreatedAt.Unix(),
171 // Todo: this might not be the initial title, we need to query the
172 // timeline to be sure
173 issue.Title,
174 cleanupText(string(*edit.Diff)),
175 nil,
176 map[string]string{
177 keyGithubId: parseId(issue.Id),
178 keyGithubUrl: issue.Url.String(),
179 },
180 )
181 if err != nil {
182 return nil, err
183 }
184 continue
185 }
186
187 target, err := b.ResolveTargetWithMetadata(keyGithubId, parseId(issue.Id))
188 if err != nil {
189 return nil, err
190 }
191
192 err = gi.ensureCommentEdit(b, target, edit)
193 if err != nil {
194 return nil, err
195 }
196 }
197
198 if !issue.UserContentEdits.PageInfo.HasNextPage {
199 // if we still didn't get a legit edit, create the bug from the issue data
200 if b == nil {
201 return repo.NewBugRaw(
202 gi.makePerson(issue.Author),
203 issue.CreatedAt.Unix(),
204 // Todo: this might not be the initial title, we need to query the
205 // timeline to be sure
206 issue.Title,
207 cleanupText(string(issue.Body)),
208 nil,
209 map[string]string{
210 keyGithubId: parseId(issue.Id),
211 keyGithubUrl: issue.Url.String(),
212 },
213 )
214 }
215 return b, nil
216 }
217
218 // We have more edit, querying them
219
220 q := &issueEditQuery{}
221 variables := map[string]interface{}{
222 "owner": rootVariables["owner"],
223 "name": rootVariables["name"],
224 "issueFirst": rootVariables["issueFirst"],
225 "issueAfter": rootVariables["issueAfter"],
226 "issueEditLast": githubv4.Int(10),
227 "issueEditBefore": issue.UserContentEdits.PageInfo.StartCursor,
228 }
229
230 for {
231 err := gi.client.Query(context.TODO(), &q, variables)
232 if err != nil {
233 return nil, err
234 }
235
236 edits := q.Repository.Issues.Nodes[0].UserContentEdits
237
238 if len(edits.Nodes) == 0 {
239 return b, nil
240 }
241
242 for _, edit := range edits.Nodes {
243 if b == nil {
244 if edit.Diff == nil {
245 // not enough data given by github for old edit, ignore them
246 continue
247 }
248
249 // we create the bug as soon as we have a legit first edition
250 b, err = repo.NewBugRaw(
251 gi.makePerson(issue.Author),
252 issue.CreatedAt.Unix(),
253 // Todo: this might not be the initial title, we need to query the
254 // timeline to be sure
255 issue.Title,
256 cleanupText(string(*edit.Diff)),
257 nil,
258 map[string]string{
259 keyGithubId: parseId(issue.Id),
260 keyGithubUrl: issue.Url.String(),
261 },
262 )
263 if err != nil {
264 return nil, err
265 }
266 continue
267 }
268
269 target, err := b.ResolveTargetWithMetadata(keyGithubId, parseId(issue.Id))
270 if err != nil {
271 return nil, err
272 }
273
274 err = gi.ensureCommentEdit(b, target, edit)
275 if err != nil {
276 return nil, err
277 }
278 }
279
280 if !edits.PageInfo.HasNextPage {
281 break
282 }
283
284 variables["issueEditBefore"] = edits.PageInfo.StartCursor
285 }
286
287 // TODO: check + import files
288
289 // if we still didn't get a legit edit, create the bug from the issue data
290 if b == nil {
291 return repo.NewBugRaw(
292 gi.makePerson(issue.Author),
293 issue.CreatedAt.Unix(),
294 // Todo: this might not be the initial title, we need to query the
295 // timeline to be sure
296 issue.Title,
297 cleanupText(string(issue.Body)),
298 nil,
299 map[string]string{
300 keyGithubId: parseId(issue.Id),
301 keyGithubUrl: issue.Url.String(),
302 },
303 )
304 }
305
306 return b, nil
307}
308
309func (gi *githubImporter) ensureTimelineItem(b *cache.BugCache, cursor githubv4.String, item timelineItem, rootVariables map[string]interface{}) error {
310 fmt.Printf("import %s\n", item.Typename)
311
312 switch item.Typename {
313 case "IssueComment":
314 return gi.ensureComment(b, cursor, item.IssueComment, rootVariables)
315
316 case "LabeledEvent":
317 id := parseId(item.LabeledEvent.Id)
318 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
319 if err != cache.ErrNoMatchingOp {
320 return err
321 }
322 _, err = b.ChangeLabelsRaw(
323 gi.makePerson(item.LabeledEvent.Actor),
324 item.LabeledEvent.CreatedAt.Unix(),
325 []string{
326 string(item.LabeledEvent.Label.Name),
327 },
328 nil,
329 map[string]string{keyGithubId: id},
330 )
331 return err
332
333 case "UnlabeledEvent":
334 id := parseId(item.UnlabeledEvent.Id)
335 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
336 if err != cache.ErrNoMatchingOp {
337 return err
338 }
339 _, err = b.ChangeLabelsRaw(
340 gi.makePerson(item.UnlabeledEvent.Actor),
341 item.UnlabeledEvent.CreatedAt.Unix(),
342 nil,
343 []string{
344 string(item.UnlabeledEvent.Label.Name),
345 },
346 map[string]string{keyGithubId: id},
347 )
348 return err
349
350 case "ClosedEvent":
351 id := parseId(item.ClosedEvent.Id)
352 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
353 if err != cache.ErrNoMatchingOp {
354 return err
355 }
356 return b.CloseRaw(
357 gi.makePerson(item.ClosedEvent.Actor),
358 item.ClosedEvent.CreatedAt.Unix(),
359 map[string]string{keyGithubId: id},
360 )
361
362 case "ReopenedEvent":
363 id := parseId(item.ReopenedEvent.Id)
364 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
365 if err != cache.ErrNoMatchingOp {
366 return err
367 }
368 return b.OpenRaw(
369 gi.makePerson(item.ReopenedEvent.Actor),
370 item.ReopenedEvent.CreatedAt.Unix(),
371 map[string]string{keyGithubId: id},
372 )
373
374 case "RenamedTitleEvent":
375 id := parseId(item.RenamedTitleEvent.Id)
376 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
377 if err != cache.ErrNoMatchingOp {
378 return err
379 }
380 return b.SetTitleRaw(
381 gi.makePerson(item.RenamedTitleEvent.Actor),
382 item.RenamedTitleEvent.CreatedAt.Unix(),
383 string(item.RenamedTitleEvent.CurrentTitle),
384 map[string]string{keyGithubId: id},
385 )
386
387 default:
388 fmt.Println("ignore event ", item.Typename)
389 }
390
391 return nil
392}
393
394func (gi *githubImporter) ensureComment(b *cache.BugCache, cursor githubv4.String, comment issueComment, rootVariables map[string]interface{}) error {
395 target, err := b.ResolveTargetWithMetadata(keyGithubId, parseId(comment.Id))
396 if err != nil && err != cache.ErrNoMatchingOp {
397 // real error
398 return err
399 }
400
401 // if there is no edit, the UserContentEdits given by github is empty. That
402 // means that the original message is given by the comment message.
403 //
404 // if there is edits, the UserContentEdits given by github contains both the
405 // original message and the following edits. The comment message give the last
406 // version so we don't care about that.
407 //
408 // the tricky part: for a comment older than the UserContentEdits API, github
409 // doesn't have the previous message version anymore and give an edition
410 // with .Diff == nil. We have to filter them.
411
412 if len(comment.UserContentEdits.Nodes) == 0 {
413 if err == cache.ErrNoMatchingOp {
414 err = b.AddCommentRaw(
415 gi.makePerson(comment.Author),
416 comment.CreatedAt.Unix(),
417 cleanupText(string(comment.Body)),
418 nil,
419 map[string]string{
420 keyGithubId: parseId(comment.Id),
421 },
422 )
423
424 if err != nil {
425 return err
426 }
427 }
428
429 return nil
430 }
431
432 // reverse the order, because github
433 reverseEdits(comment.UserContentEdits.Nodes)
434
435 for i, edit := range comment.UserContentEdits.Nodes {
436 if target != "" && i == 0 {
437 // The first edit in the github result is the comment creation itself, we already have that
438 continue
439 }
440
441 if target == "" {
442 if edit.Diff == nil {
443 // not enough data given by github for old edit, ignore them
444 continue
445 }
446
447 err = b.AddCommentRaw(
448 gi.makePerson(comment.Author),
449 comment.CreatedAt.Unix(),
450 cleanupText(string(*edit.Diff)),
451 nil,
452 map[string]string{
453 keyGithubId: parseId(comment.Id),
454 keyGithubUrl: comment.Url.String(),
455 },
456 )
457 if err != nil {
458 return err
459 }
460 }
461
462 err := gi.ensureCommentEdit(b, target, edit)
463 if err != nil {
464 return err
465 }
466 }
467
468 if !comment.UserContentEdits.PageInfo.HasNextPage {
469 return nil
470 }
471
472 // We have more edit, querying them
473
474 q := &commentEditQuery{}
475 variables := map[string]interface{}{
476 "owner": rootVariables["owner"],
477 "name": rootVariables["name"],
478 "issueFirst": rootVariables["issueFirst"],
479 "issueAfter": rootVariables["issueAfter"],
480 "timelineFirst": githubv4.Int(1),
481 "timelineAfter": cursor,
482 "commentEditLast": githubv4.Int(10),
483 "commentEditBefore": comment.UserContentEdits.PageInfo.StartCursor,
484 }
485
486 for {
487 err := gi.client.Query(context.TODO(), &q, variables)
488 if err != nil {
489 return err
490 }
491
492 edits := q.Repository.Issues.Nodes[0].Timeline.Nodes[0].IssueComment.UserContentEdits
493
494 if len(edits.Nodes) == 0 {
495 return nil
496 }
497
498 for i, edit := range edits.Nodes {
499 if i == 0 {
500 // The first edit in the github result is the creation itself, we already have that
501 continue
502 }
503
504 err := gi.ensureCommentEdit(b, target, edit)
505 if err != nil {
506 return err
507 }
508 }
509
510 if !edits.PageInfo.HasNextPage {
511 break
512 }
513
514 variables["commentEditBefore"] = edits.PageInfo.StartCursor
515 }
516
517 // TODO: check + import files
518
519 return nil
520}
521
522func (gi *githubImporter) ensureCommentEdit(b *cache.BugCache, target git.Hash, edit userContentEdit) error {
523 if edit.Diff == nil {
524 // this happen if the event is older than early 2018, Github doesn't have the data before that.
525 // Best we can do is to ignore the event.
526 return nil
527 }
528
529 if edit.Editor == nil {
530 return fmt.Errorf("no editor")
531 }
532
533 _, err := b.ResolveTargetWithMetadata(keyGithubId, parseId(edit.Id))
534 if err == nil {
535 // already imported
536 return nil
537 }
538 if err != cache.ErrNoMatchingOp {
539 // real error
540 return err
541 }
542
543 fmt.Println("import edition")
544
545 switch {
546 case edit.DeletedAt != nil:
547 // comment deletion, not supported yet
548
549 case edit.DeletedAt == nil:
550 // comment edition
551 err := b.EditCommentRaw(
552 gi.makePerson(edit.Editor),
553 edit.CreatedAt.Unix(),
554 target,
555 cleanupText(string(*edit.Diff)),
556 map[string]string{
557 keyGithubId: parseId(edit.Id),
558 },
559 )
560 if err != nil {
561 return err
562 }
563 }
564
565 return nil
566}
567
568// makePerson create a bug.Person from the Github data
569func (gi *githubImporter) makePerson(actor *actor) identity.Interface {
570 if actor == nil {
571 return gi.ghost
572 }
573 var name string
574 var email string
575
576 switch actor.Typename {
577 case "User":
578 if actor.User.Name != nil {
579 name = string(*(actor.User.Name))
580 }
581 email = string(actor.User.Email)
582 case "Organization":
583 if actor.Organization.Name != nil {
584 name = string(*(actor.Organization.Name))
585 }
586 if actor.Organization.Email != nil {
587 email = string(*(actor.Organization.Email))
588 }
589 case "Bot":
590 }
591
592 return bug.Person{
593 Name: name,
594 Email: email,
595 Login: string(actor.Login),
596 AvatarUrl: string(actor.AvatarUrl),
597 }
598}
599
600func (gi *githubImporter) fetchGhost() error {
601 var q userQuery
602
603 variables := map[string]interface{}{
604 "login": githubv4.String("ghost"),
605 }
606
607 err := gi.client.Query(context.TODO(), &q, variables)
608 if err != nil {
609 return err
610 }
611
612 var name string
613 if q.User.Name != nil {
614 name = string(*q.User.Name)
615 }
616
617 gi.ghost = identity.NewIdentityFull(
618 name,
619 string(q.User.Login),
620 string(q.User.AvatarUrl),
621 string(q.User.Email),
622 )
623
624 return nil
625}
626
627// parseId convert the unusable githubv4.ID (an interface{}) into a string
628func parseId(id githubv4.ID) string {
629 return fmt.Sprintf("%v", id)
630}
631
632func cleanupText(text string) string {
633 // windows new line, Github, really ?
634 text = strings.Replace(text, "\r\n", "\n", -1)
635
636 // trim extra new line not displayed in the github UI but still present in the data
637 return strings.TrimSpace(text)
638}
639
640func reverseEdits(edits []userContentEdit) []userContentEdit {
641 for i, j := 0, len(edits)-1; i < j; i, j = i+1, j-1 {
642 edits[i], edits[j] = edits[j], edits[i]
643 }
644 return edits
645}