1package github
2
3import (
4 "context"
5 "fmt"
6 "strings"
7
8 "github.com/MichaelMure/git-bug/bridge/core"
9 "github.com/MichaelMure/git-bug/bug"
10 "github.com/MichaelMure/git-bug/cache"
11 "github.com/MichaelMure/git-bug/util/git"
12 "github.com/shurcooL/githubv4"
13)
14
15const keyGithubId = "github-id"
16const keyGithubUrl = "github-url"
17
18// githubImporter implement the Importer interface
19type githubImporter struct {
20 client *githubv4.Client
21 conf core.Configuration
22 ghost bug.Person
23}
24
25func (gi *githubImporter) Init(conf core.Configuration) error {
26 gi.conf = conf
27 gi.client = buildClient(conf)
28
29 return gi.fetchGhost()
30}
31
32func (gi *githubImporter) ImportAll(repo *cache.RepoCache) error {
33 q := &issueTimelineQuery{}
34 variables := map[string]interface{}{
35 "owner": githubv4.String(gi.conf[keyUser]),
36 "name": githubv4.String(gi.conf[keyProject]),
37 "issueFirst": githubv4.Int(1),
38 "issueAfter": (*githubv4.String)(nil),
39 "timelineFirst": githubv4.Int(10),
40 "timelineAfter": (*githubv4.String)(nil),
41
42 // Fun fact, github provide the comment edition in reverse chronological
43 // order, because haha. Look at me, I'm dying of laughter.
44 "issueEditLast": githubv4.Int(10),
45 "issueEditBefore": (*githubv4.String)(nil),
46 "commentEditLast": githubv4.Int(10),
47 "commentEditBefore": (*githubv4.String)(nil),
48 }
49
50 var b *cache.BugCache
51
52 for {
53 err := gi.client.Query(context.TODO(), &q, variables)
54 if err != nil {
55 return err
56 }
57
58 if len(q.Repository.Issues.Nodes) == 0 {
59 return nil
60 }
61
62 issue := q.Repository.Issues.Nodes[0]
63
64 if b == nil {
65 b, err = gi.ensureIssue(repo, issue, variables)
66 if err != nil {
67 return err
68 }
69 }
70
71 for _, itemEdge := range q.Repository.Issues.Nodes[0].Timeline.Edges {
72 gi.ensureTimelineItem(b, itemEdge.Cursor, itemEdge.Node, variables)
73 }
74
75 if !issue.Timeline.PageInfo.HasNextPage {
76 err = b.CommitAsNeeded()
77 if err != nil {
78 return err
79 }
80
81 b = nil
82
83 if !q.Repository.Issues.PageInfo.HasNextPage {
84 break
85 }
86
87 variables["issueAfter"] = githubv4.NewString(q.Repository.Issues.PageInfo.EndCursor)
88 variables["timelineAfter"] = (*githubv4.String)(nil)
89 continue
90 }
91
92 variables["timelineAfter"] = githubv4.NewString(issue.Timeline.PageInfo.EndCursor)
93 }
94
95 return nil
96}
97
98func (gi *githubImporter) Import(repo *cache.RepoCache, id string) error {
99 fmt.Println("IMPORT")
100
101 return nil
102}
103
104func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline, rootVariables map[string]interface{}) (*cache.BugCache, error) {
105 fmt.Printf("import issue: %s\n", issue.Title)
106
107 b, err := repo.ResolveBugCreateMetadata(keyGithubId, parseId(issue.Id))
108 if err != nil && err != bug.ErrBugNotExist {
109 return nil, err
110 }
111
112 // if there is no edit, the UserContentEdits given by github is empty. That
113 // means that the original message is given by the issue message.
114 //
115 // if there is edits, the UserContentEdits given by github contains both the
116 // original message and the following edits. The issue message give the last
117 // version so we don't care about that.
118 //
119 // the tricky part: for an issue older than the UserContentEdits API, github
120 // doesn't have the previous message version anymore and give an edition
121 // with .Diff == nil. We have to filter them.
122
123 if len(issue.UserContentEdits.Nodes) == 0 {
124 if err == bug.ErrBugNotExist {
125 b, err = repo.NewBugRaw(
126 gi.makePerson(issue.Author),
127 issue.CreatedAt.Unix(),
128 // Todo: this might not be the initial title, we need to query the
129 // timeline to be sure
130 issue.Title,
131 cleanupText(string(issue.Body)),
132 nil,
133 map[string]string{
134 keyGithubId: parseId(issue.Id),
135 keyGithubUrl: issue.Url.String(),
136 },
137 )
138
139 if err != nil {
140 return nil, err
141 }
142 }
143
144 return b, nil
145 }
146
147 // reverse the order, because github
148 reverseEdits(issue.UserContentEdits.Nodes)
149
150 for i, edit := range issue.UserContentEdits.Nodes {
151 if b != nil && i == 0 {
152 // The first edit in the github result is the creation itself, we already have that
153 continue
154 }
155
156 if b == nil {
157 if edit.Diff == nil {
158 // not enough data given by github for old edit, ignore them
159 continue
160 }
161
162 // we create the bug as soon as we have a legit first edition
163 b, err = repo.NewBugRaw(
164 gi.makePerson(issue.Author),
165 issue.CreatedAt.Unix(),
166 // Todo: this might not be the initial title, we need to query the
167 // timeline to be sure
168 issue.Title,
169 cleanupText(string(*edit.Diff)),
170 nil,
171 map[string]string{
172 keyGithubId: parseId(issue.Id),
173 keyGithubUrl: issue.Url.String(),
174 },
175 )
176 if err != nil {
177 return nil, err
178 }
179 continue
180 }
181
182 target, err := b.ResolveTargetWithMetadata(keyGithubId, parseId(issue.Id))
183 if err != nil {
184 return nil, err
185 }
186
187 err = gi.ensureCommentEdit(b, target, edit)
188 if err != nil {
189 return nil, err
190 }
191 }
192
193 if !issue.UserContentEdits.PageInfo.HasNextPage {
194 // if we still didn't get a legit edit, create the bug from the issue data
195 if b == nil {
196 return repo.NewBugRaw(
197 gi.makePerson(issue.Author),
198 issue.CreatedAt.Unix(),
199 // Todo: this might not be the initial title, we need to query the
200 // timeline to be sure
201 issue.Title,
202 cleanupText(string(issue.Body)),
203 nil,
204 map[string]string{
205 keyGithubId: parseId(issue.Id),
206 keyGithubUrl: issue.Url.String(),
207 },
208 )
209 }
210 return b, nil
211 }
212
213 // We have more edit, querying them
214
215 q := &issueEditQuery{}
216 variables := map[string]interface{}{
217 "owner": rootVariables["owner"],
218 "name": rootVariables["name"],
219 "issueFirst": rootVariables["issueFirst"],
220 "issueAfter": rootVariables["issueAfter"],
221 "issueEditLast": githubv4.Int(10),
222 "issueEditBefore": issue.UserContentEdits.PageInfo.StartCursor,
223 }
224
225 for {
226 err := gi.client.Query(context.TODO(), &q, variables)
227 if err != nil {
228 return nil, err
229 }
230
231 edits := q.Repository.Issues.Nodes[0].UserContentEdits
232
233 if len(edits.Nodes) == 0 {
234 return b, nil
235 }
236
237 for _, edit := range edits.Nodes {
238 if b == nil {
239 if edit.Diff == nil {
240 // not enough data given by github for old edit, ignore them
241 continue
242 }
243
244 // we create the bug as soon as we have a legit first edition
245 b, err = repo.NewBugRaw(
246 gi.makePerson(issue.Author),
247 issue.CreatedAt.Unix(),
248 // Todo: this might not be the initial title, we need to query the
249 // timeline to be sure
250 issue.Title,
251 cleanupText(string(*edit.Diff)),
252 nil,
253 map[string]string{
254 keyGithubId: parseId(issue.Id),
255 keyGithubUrl: issue.Url.String(),
256 },
257 )
258 if err != nil {
259 return nil, err
260 }
261 continue
262 }
263
264 target, err := b.ResolveTargetWithMetadata(keyGithubId, parseId(issue.Id))
265 if err != nil {
266 return nil, err
267 }
268
269 err = gi.ensureCommentEdit(b, target, edit)
270 if err != nil {
271 return nil, err
272 }
273 }
274
275 if !edits.PageInfo.HasNextPage {
276 break
277 }
278
279 variables["issueEditBefore"] = edits.PageInfo.StartCursor
280 }
281
282 // TODO: check + import files
283
284 // if we still didn't get a legit edit, create the bug from the issue data
285 if b == nil {
286 return repo.NewBugRaw(
287 gi.makePerson(issue.Author),
288 issue.CreatedAt.Unix(),
289 // Todo: this might not be the initial title, we need to query the
290 // timeline to be sure
291 issue.Title,
292 cleanupText(string(issue.Body)),
293 nil,
294 map[string]string{
295 keyGithubId: parseId(issue.Id),
296 keyGithubUrl: issue.Url.String(),
297 },
298 )
299 }
300
301 return b, nil
302}
303
304func (gi *githubImporter) ensureTimelineItem(b *cache.BugCache, cursor githubv4.String, item timelineItem, rootVariables map[string]interface{}) error {
305 fmt.Printf("import %s\n", item.Typename)
306
307 switch item.Typename {
308 case "IssueComment":
309 return gi.ensureComment(b, cursor, item.IssueComment, rootVariables)
310
311 case "LabeledEvent":
312 id := parseId(item.LabeledEvent.Id)
313 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
314 if err != cache.ErrNoMatchingOp {
315 return err
316 }
317 _, err = b.ChangeLabelsRaw(
318 gi.makePerson(item.LabeledEvent.Actor),
319 item.LabeledEvent.CreatedAt.Unix(),
320 []string{
321 string(item.LabeledEvent.Label.Name),
322 },
323 nil,
324 map[string]string{keyGithubId: id},
325 )
326 return err
327
328 case "UnlabeledEvent":
329 id := parseId(item.UnlabeledEvent.Id)
330 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
331 if err != cache.ErrNoMatchingOp {
332 return err
333 }
334 _, err = b.ChangeLabelsRaw(
335 gi.makePerson(item.UnlabeledEvent.Actor),
336 item.UnlabeledEvent.CreatedAt.Unix(),
337 nil,
338 []string{
339 string(item.UnlabeledEvent.Label.Name),
340 },
341 map[string]string{keyGithubId: id},
342 )
343 return err
344
345 case "ClosedEvent":
346 id := parseId(item.ClosedEvent.Id)
347 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
348 if err != cache.ErrNoMatchingOp {
349 return err
350 }
351 return b.CloseRaw(
352 gi.makePerson(item.ClosedEvent.Actor),
353 item.ClosedEvent.CreatedAt.Unix(),
354 map[string]string{keyGithubId: id},
355 )
356
357 case "ReopenedEvent":
358 id := parseId(item.ReopenedEvent.Id)
359 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
360 if err != cache.ErrNoMatchingOp {
361 return err
362 }
363 return b.OpenRaw(
364 gi.makePerson(item.ReopenedEvent.Actor),
365 item.ReopenedEvent.CreatedAt.Unix(),
366 map[string]string{keyGithubId: id},
367 )
368
369 case "RenamedTitleEvent":
370 id := parseId(item.RenamedTitleEvent.Id)
371 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
372 if err != cache.ErrNoMatchingOp {
373 return err
374 }
375 return b.SetTitleRaw(
376 gi.makePerson(item.RenamedTitleEvent.Actor),
377 item.RenamedTitleEvent.CreatedAt.Unix(),
378 string(item.RenamedTitleEvent.CurrentTitle),
379 map[string]string{keyGithubId: id},
380 )
381
382 default:
383 fmt.Println("ignore event ", item.Typename)
384 }
385
386 return nil
387}
388
389func (gi *githubImporter) ensureComment(b *cache.BugCache, cursor githubv4.String, comment issueComment, rootVariables map[string]interface{}) error {
390 target, err := b.ResolveTargetWithMetadata(keyGithubId, parseId(comment.Id))
391 if err != nil && err != cache.ErrNoMatchingOp {
392 // real error
393 return err
394 }
395
396 // if there is no edit, the UserContentEdits given by github is empty. That
397 // means that the original message is given by the comment message.
398 //
399 // if there is edits, the UserContentEdits given by github contains both the
400 // original message and the following edits. The comment message give the last
401 // version so we don't care about that.
402 //
403 // the tricky part: for a comment older than the UserContentEdits API, github
404 // doesn't have the previous message version anymore and give an edition
405 // with .Diff == nil. We have to filter them.
406
407 if len(comment.UserContentEdits.Nodes) == 0 {
408 if err == cache.ErrNoMatchingOp {
409 err = b.AddCommentRaw(
410 gi.makePerson(comment.Author),
411 comment.CreatedAt.Unix(),
412 cleanupText(string(comment.Body)),
413 nil,
414 map[string]string{
415 keyGithubId: parseId(comment.Id),
416 },
417 )
418
419 if err != nil {
420 return err
421 }
422 }
423
424 return nil
425 }
426
427 // reverse the order, because github
428 reverseEdits(comment.UserContentEdits.Nodes)
429
430 for i, edit := range comment.UserContentEdits.Nodes {
431 if target != "" && i == 0 {
432 // The first edit in the github result is the comment creation itself, we already have that
433 continue
434 }
435
436 if target == "" {
437 if edit.Diff == nil {
438 // not enough data given by github for old edit, ignore them
439 continue
440 }
441
442 err = b.AddCommentRaw(
443 gi.makePerson(comment.Author),
444 comment.CreatedAt.Unix(),
445 cleanupText(string(*edit.Diff)),
446 nil,
447 map[string]string{
448 keyGithubId: parseId(comment.Id),
449 keyGithubUrl: comment.Url.String(),
450 },
451 )
452 if err != nil {
453 return err
454 }
455 }
456
457 err := gi.ensureCommentEdit(b, target, edit)
458 if err != nil {
459 return err
460 }
461 }
462
463 if !comment.UserContentEdits.PageInfo.HasNextPage {
464 return nil
465 }
466
467 // We have more edit, querying them
468
469 q := &commentEditQuery{}
470 variables := map[string]interface{}{
471 "owner": rootVariables["owner"],
472 "name": rootVariables["name"],
473 "issueFirst": rootVariables["issueFirst"],
474 "issueAfter": rootVariables["issueAfter"],
475 "timelineFirst": githubv4.Int(1),
476 "timelineAfter": cursor,
477 "commentEditLast": githubv4.Int(10),
478 "commentEditBefore": comment.UserContentEdits.PageInfo.StartCursor,
479 }
480
481 for {
482 err := gi.client.Query(context.TODO(), &q, variables)
483 if err != nil {
484 return err
485 }
486
487 edits := q.Repository.Issues.Nodes[0].Timeline.Nodes[0].IssueComment.UserContentEdits
488
489 if len(edits.Nodes) == 0 {
490 return nil
491 }
492
493 for i, edit := range edits.Nodes {
494 if i == 0 {
495 // The first edit in the github result is the creation itself, we already have that
496 continue
497 }
498
499 err := gi.ensureCommentEdit(b, target, edit)
500 if err != nil {
501 return err
502 }
503 }
504
505 if !edits.PageInfo.HasNextPage {
506 break
507 }
508
509 variables["commentEditBefore"] = edits.PageInfo.StartCursor
510 }
511
512 // TODO: check + import files
513
514 return nil
515}
516
517func (gi *githubImporter) ensureCommentEdit(b *cache.BugCache, target git.Hash, edit userContentEdit) error {
518 if edit.Diff == nil {
519 // this happen if the event is older than early 2018, Github doesn't have the data before that.
520 // Best we can do is to ignore the event.
521 return nil
522 }
523
524 if edit.Editor == nil {
525 return fmt.Errorf("no editor")
526 }
527
528 _, err := b.ResolveTargetWithMetadata(keyGithubId, parseId(edit.Id))
529 if err == nil {
530 // already imported
531 return nil
532 }
533 if err != cache.ErrNoMatchingOp {
534 // real error
535 return err
536 }
537
538 fmt.Println("import edition")
539
540 switch {
541 case edit.DeletedAt != nil:
542 // comment deletion, not supported yet
543
544 case edit.DeletedAt == nil:
545 // comment edition
546 err := b.EditCommentRaw(
547 gi.makePerson(edit.Editor),
548 edit.CreatedAt.Unix(),
549 target,
550 cleanupText(string(*edit.Diff)),
551 map[string]string{
552 keyGithubId: parseId(edit.Id),
553 },
554 )
555 if err != nil {
556 return err
557 }
558 }
559
560 return nil
561}
562
563// makePerson create a bug.Person from the Github data
564func (gi *githubImporter) makePerson(actor *actor) bug.Person {
565 if actor == nil {
566 return gi.ghost
567 }
568
569 return bug.Person{
570 Name: string(actor.Login),
571 AvatarUrl: string(actor.AvatarUrl),
572 }
573}
574
575func (gi *githubImporter) fetchGhost() error {
576 var q userQuery
577
578 variables := map[string]interface{}{
579 "login": githubv4.String("ghost"),
580 }
581
582 err := gi.client.Query(context.TODO(), &q, variables)
583 if err != nil {
584 return err
585 }
586
587 gi.ghost = bug.Person{
588 Name: string(q.User.Login),
589 AvatarUrl: string(q.User.AvatarUrl),
590 }
591
592 return nil
593}
594
595// parseId convert the unusable githubv4.ID (an interface{}) into a string
596func parseId(id githubv4.ID) string {
597 return fmt.Sprintf("%v", id)
598}
599
600func cleanupText(text string) string {
601 // windows new line, Github, really ?
602 text = strings.Replace(text, "\r\n", "\n", -1)
603
604 // trim extra new line not displayed in the github UI but still present in the data
605 return strings.TrimSpace(text)
606}
607
608func reverseEdits(edits []userContentEdit) []userContentEdit {
609 for i, j := 0, len(edits)-1; i < j; i, j = i+1, j-1 {
610 edits[i], edits[j] = edits[j], edits[i]
611 }
612 return edits
613}