1package github
2
3import (
4 "context"
5 "fmt"
6 "strings"
7
8 "github.com/MichaelMure/git-bug/bridge/core"
9 "github.com/MichaelMure/git-bug/bug"
10 "github.com/MichaelMure/git-bug/cache"
11 "github.com/MichaelMure/git-bug/util/git"
12 "github.com/shurcooL/githubv4"
13)
14
15const keyGithubId = "github-id"
16const keyGithubUrl = "github-url"
17
18// githubImporter implement the Importer interface
19type githubImporter struct{}
20
21func (*githubImporter) ImportAll(repo *cache.RepoCache, conf core.Configuration) error {
22 client := buildClient(conf)
23
24 q := &issueTimelineQuery{}
25 variables := map[string]interface{}{
26 "owner": githubv4.String(conf[keyUser]),
27 "name": githubv4.String(conf[keyProject]),
28 "issueFirst": githubv4.Int(1),
29 "issueAfter": (*githubv4.String)(nil),
30 "timelineFirst": githubv4.Int(10),
31 "timelineAfter": (*githubv4.String)(nil),
32
33 // Fun fact, github provide the comment edition in reverse chronological
34 // order, because haha. Look at me, I'm dying of laughter.
35 "issueEditLast": githubv4.Int(10),
36 "issueEditBefore": (*githubv4.String)(nil),
37 "commentEditLast": githubv4.Int(10),
38 "commentEditBefore": (*githubv4.String)(nil),
39 }
40
41 var b *cache.BugCache
42
43 for {
44 err := client.Query(context.TODO(), &q, variables)
45 if err != nil {
46 return err
47 }
48
49 if len(q.Repository.Issues.Nodes) == 0 {
50 return nil
51 }
52
53 issue := q.Repository.Issues.Nodes[0]
54
55 if b == nil {
56 b, err = ensureIssue(repo, issue, client, variables)
57 if err != nil {
58 return err
59 }
60 }
61
62 for _, itemEdge := range q.Repository.Issues.Nodes[0].Timeline.Edges {
63 ensureTimelineItem(b, itemEdge.Cursor, itemEdge.Node, client, variables)
64 }
65
66 if !issue.Timeline.PageInfo.HasNextPage {
67 err = b.CommitAsNeeded()
68 if err != nil {
69 return err
70 }
71
72 b = nil
73
74 if !q.Repository.Issues.PageInfo.HasNextPage {
75 break
76 }
77
78 variables["issueAfter"] = githubv4.NewString(q.Repository.Issues.PageInfo.EndCursor)
79 variables["timelineAfter"] = (*githubv4.String)(nil)
80 continue
81 }
82
83 variables["timelineAfter"] = githubv4.NewString(issue.Timeline.PageInfo.EndCursor)
84 }
85
86 return nil
87}
88
89func (*githubImporter) Import(repo *cache.RepoCache, conf core.Configuration, id string) error {
90 fmt.Println(conf)
91 fmt.Println("IMPORT")
92
93 return nil
94}
95
96func ensureIssue(repo *cache.RepoCache, issue issueTimeline, client *githubv4.Client, rootVariables map[string]interface{}) (*cache.BugCache, error) {
97 fmt.Printf("import issue: %s\n", issue.Title)
98
99 b, err := repo.ResolveBugCreateMetadata(keyGithubId, parseId(issue.Id))
100 if err != nil && err != bug.ErrBugNotExist {
101 return nil, err
102 }
103
104 // if there is no edit, the UserContentEdits given by github is empty. That
105 // means that the original message is given by the issue message.
106 //
107 // if there is edits, the UserContentEdits given by github contains both the
108 // original message and the following edits. The issue message give the last
109 // version so we don't care about that.
110 //
111 // the tricky part: for an issue older than the UserContentEdits API, github
112 // doesn't have the previous message version anymore and give an edition
113 // with .Diff == nil. We have to filter them.
114
115 if len(issue.UserContentEdits.Nodes) == 0 {
116 if err == bug.ErrBugNotExist {
117 b, err = repo.NewBugRaw(
118 makePerson(issue.Author),
119 issue.CreatedAt.Unix(),
120 // Todo: this might not be the initial title, we need to query the
121 // timeline to be sure
122 issue.Title,
123 cleanupText(string(issue.Body)),
124 nil,
125 map[string]string{
126 keyGithubId: parseId(issue.Id),
127 keyGithubUrl: issue.Url.String(),
128 },
129 )
130
131 if err != nil {
132 return nil, err
133 }
134 }
135
136 return b, nil
137 }
138
139 // reverse the order, because github
140 reverseEdits(issue.UserContentEdits.Nodes)
141
142 for i, edit := range issue.UserContentEdits.Nodes {
143 if b != nil && i == 0 {
144 // The first edit in the github result is the creation itself, we already have that
145 continue
146 }
147
148 if b == nil {
149 if edit.Diff == nil {
150 // not enough data given by github for old edit, ignore them
151 continue
152 }
153
154 // we create the bug as soon as we have a legit first edition
155 b, err = repo.NewBugRaw(
156 makePerson(issue.Author),
157 issue.CreatedAt.Unix(),
158 // Todo: this might not be the initial title, we need to query the
159 // timeline to be sure
160 issue.Title,
161 cleanupText(string(*edit.Diff)),
162 nil,
163 map[string]string{
164 keyGithubId: parseId(issue.Id),
165 keyGithubUrl: issue.Url.String(),
166 },
167 )
168 if err != nil {
169 return nil, err
170 }
171 continue
172 }
173
174 target, err := b.ResolveTargetWithMetadata(keyGithubId, parseId(issue.Id))
175 if err != nil {
176 return nil, err
177 }
178
179 err = ensureCommentEdit(b, target, edit)
180 if err != nil {
181 return nil, err
182 }
183 }
184
185 if !issue.UserContentEdits.PageInfo.HasNextPage {
186 // if we still didn't get a legit edit, create the bug from the issue data
187 if b == nil {
188 return repo.NewBugRaw(
189 makePerson(issue.Author),
190 issue.CreatedAt.Unix(),
191 // Todo: this might not be the initial title, we need to query the
192 // timeline to be sure
193 issue.Title,
194 cleanupText(string(issue.Body)),
195 nil,
196 map[string]string{
197 keyGithubId: parseId(issue.Id),
198 keyGithubUrl: issue.Url.String(),
199 },
200 )
201 }
202 return b, nil
203 }
204
205 // We have more edit, querying them
206
207 q := &issueEditQuery{}
208 variables := map[string]interface{}{
209 "owner": rootVariables["owner"],
210 "name": rootVariables["name"],
211 "issueFirst": rootVariables["issueFirst"],
212 "issueAfter": rootVariables["issueAfter"],
213 "issueEditLast": githubv4.Int(10),
214 "issueEditBefore": issue.UserContentEdits.PageInfo.StartCursor,
215 }
216
217 for {
218 err := client.Query(context.TODO(), &q, variables)
219 if err != nil {
220 return nil, err
221 }
222
223 edits := q.Repository.Issues.Nodes[0].UserContentEdits
224
225 if len(edits.Nodes) == 0 {
226 return b, nil
227 }
228
229 for _, edit := range edits.Nodes {
230 if b == nil {
231 if edit.Diff == nil {
232 // not enough data given by github for old edit, ignore them
233 continue
234 }
235
236 // we create the bug as soon as we have a legit first edition
237 b, err = repo.NewBugRaw(
238 makePerson(issue.Author),
239 issue.CreatedAt.Unix(),
240 // Todo: this might not be the initial title, we need to query the
241 // timeline to be sure
242 issue.Title,
243 cleanupText(string(*edit.Diff)),
244 nil,
245 map[string]string{
246 keyGithubId: parseId(issue.Id),
247 keyGithubUrl: issue.Url.String(),
248 },
249 )
250 if err != nil {
251 return nil, err
252 }
253 continue
254 }
255
256 target, err := b.ResolveTargetWithMetadata(keyGithubId, parseId(issue.Id))
257 if err != nil {
258 return nil, err
259 }
260
261 err = ensureCommentEdit(b, target, edit)
262 if err != nil {
263 return nil, err
264 }
265 }
266
267 if !edits.PageInfo.HasNextPage {
268 break
269 }
270
271 variables["issueEditBefore"] = edits.PageInfo.StartCursor
272 }
273
274 // TODO: check + import files
275
276 // if we still didn't get a legit edit, create the bug from the issue data
277 if b == nil {
278 return repo.NewBugRaw(
279 makePerson(issue.Author),
280 issue.CreatedAt.Unix(),
281 // Todo: this might not be the initial title, we need to query the
282 // timeline to be sure
283 issue.Title,
284 cleanupText(string(issue.Body)),
285 nil,
286 map[string]string{
287 keyGithubId: parseId(issue.Id),
288 keyGithubUrl: issue.Url.String(),
289 },
290 )
291 }
292
293 return b, nil
294}
295
296func ensureTimelineItem(b *cache.BugCache, cursor githubv4.String, item timelineItem, client *githubv4.Client, rootVariables map[string]interface{}) error {
297 fmt.Printf("import %s\n", item.Typename)
298
299 switch item.Typename {
300 case "IssueComment":
301 return ensureComment(b, cursor, item.IssueComment, client, rootVariables)
302
303 case "LabeledEvent":
304 id := parseId(item.LabeledEvent.Id)
305 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
306 if err != cache.ErrNoMatchingOp {
307 return err
308 }
309 _, err = b.ChangeLabelsRaw(
310 makePerson(item.LabeledEvent.Actor),
311 item.LabeledEvent.CreatedAt.Unix(),
312 []string{
313 string(item.LabeledEvent.Label.Name),
314 },
315 nil,
316 map[string]string{keyGithubId: id},
317 )
318 return err
319
320 case "UnlabeledEvent":
321 id := parseId(item.UnlabeledEvent.Id)
322 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
323 if err != cache.ErrNoMatchingOp {
324 return err
325 }
326 _, err = b.ChangeLabelsRaw(
327 makePerson(item.UnlabeledEvent.Actor),
328 item.UnlabeledEvent.CreatedAt.Unix(),
329 nil,
330 []string{
331 string(item.UnlabeledEvent.Label.Name),
332 },
333 map[string]string{keyGithubId: id},
334 )
335 return err
336
337 case "ClosedEvent":
338 id := parseId(item.ClosedEvent.Id)
339 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
340 if err != cache.ErrNoMatchingOp {
341 return err
342 }
343 return b.CloseRaw(
344 makePerson(item.ClosedEvent.Actor),
345 item.ClosedEvent.CreatedAt.Unix(),
346 map[string]string{keyGithubId: id},
347 )
348
349 case "ReopenedEvent":
350 id := parseId(item.ReopenedEvent.Id)
351 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
352 if err != cache.ErrNoMatchingOp {
353 return err
354 }
355 return b.OpenRaw(
356 makePerson(item.ReopenedEvent.Actor),
357 item.ReopenedEvent.CreatedAt.Unix(),
358 map[string]string{keyGithubId: id},
359 )
360
361 case "RenamedTitleEvent":
362 id := parseId(item.RenamedTitleEvent.Id)
363 _, err := b.ResolveTargetWithMetadata(keyGithubId, id)
364 if err != cache.ErrNoMatchingOp {
365 return err
366 }
367 return b.SetTitleRaw(
368 makePerson(item.RenamedTitleEvent.Actor),
369 item.RenamedTitleEvent.CreatedAt.Unix(),
370 string(item.RenamedTitleEvent.CurrentTitle),
371 map[string]string{keyGithubId: id},
372 )
373
374 default:
375 fmt.Println("ignore event ", item.Typename)
376 }
377
378 return nil
379}
380
381func ensureComment(b *cache.BugCache, cursor githubv4.String, comment issueComment, client *githubv4.Client, rootVariables map[string]interface{}) error {
382 target, err := b.ResolveTargetWithMetadata(keyGithubId, parseId(comment.Id))
383 if err != nil && err != cache.ErrNoMatchingOp {
384 // real error
385 return err
386 }
387
388 // if there is no edit, the UserContentEdits given by github is empty. That
389 // means that the original message is given by the comment message.
390 //
391 // if there is edits, the UserContentEdits given by github contains both the
392 // original message and the following edits. The comment message give the last
393 // version so we don't care about that.
394 //
395 // the tricky part: for a comment older than the UserContentEdits API, github
396 // doesn't have the previous message version anymore and give an edition
397 // with .Diff == nil. We have to filter them.
398
399 if len(comment.UserContentEdits.Nodes) == 0 {
400 if err == cache.ErrNoMatchingOp {
401 err = b.AddCommentRaw(
402 makePerson(comment.Author),
403 comment.CreatedAt.Unix(),
404 cleanupText(string(comment.Body)),
405 nil,
406 map[string]string{
407 keyGithubId: parseId(comment.Id),
408 },
409 )
410
411 if err != nil {
412 return err
413 }
414 }
415
416 return nil
417 }
418
419 // reverse the order, because github
420 reverseEdits(comment.UserContentEdits.Nodes)
421
422 for i, edit := range comment.UserContentEdits.Nodes {
423 if target != "" && i == 0 {
424 // The first edit in the github result is the comment creation itself, we already have that
425 continue
426 }
427
428 if target == "" {
429 if edit.Diff == nil {
430 // not enough data given by github for old edit, ignore them
431 continue
432 }
433
434 err = b.AddCommentRaw(
435 makePerson(comment.Author),
436 comment.CreatedAt.Unix(),
437 cleanupText(string(*edit.Diff)),
438 nil,
439 map[string]string{
440 keyGithubId: parseId(comment.Id),
441 keyGithubUrl: comment.Url.String(),
442 },
443 )
444 if err != nil {
445 return err
446 }
447 }
448
449 err := ensureCommentEdit(b, target, edit)
450 if err != nil {
451 return err
452 }
453 }
454
455 if !comment.UserContentEdits.PageInfo.HasNextPage {
456 return nil
457 }
458
459 // We have more edit, querying them
460
461 q := &commentEditQuery{}
462 variables := map[string]interface{}{
463 "owner": rootVariables["owner"],
464 "name": rootVariables["name"],
465 "issueFirst": rootVariables["issueFirst"],
466 "issueAfter": rootVariables["issueAfter"],
467 "timelineFirst": githubv4.Int(1),
468 "timelineAfter": cursor,
469 "commentEditLast": githubv4.Int(10),
470 "commentEditBefore": comment.UserContentEdits.PageInfo.StartCursor,
471 }
472
473 for {
474 err := client.Query(context.TODO(), &q, variables)
475 if err != nil {
476 return err
477 }
478
479 edits := q.Repository.Issues.Nodes[0].Timeline.Nodes[0].IssueComment.UserContentEdits
480
481 if len(edits.Nodes) == 0 {
482 return nil
483 }
484
485 for i, edit := range edits.Nodes {
486 if i == 0 {
487 // The first edit in the github result is the creation itself, we already have that
488 continue
489 }
490
491 err := ensureCommentEdit(b, target, edit)
492 if err != nil {
493 return err
494 }
495 }
496
497 if !edits.PageInfo.HasNextPage {
498 break
499 }
500
501 variables["commentEditBefore"] = edits.PageInfo.StartCursor
502 }
503
504 // TODO: check + import files
505
506 return nil
507}
508
509func ensureCommentEdit(b *cache.BugCache, target git.Hash, edit userContentEdit) error {
510 if edit.Diff == nil {
511 // this happen if the event is older than early 2018, Github doesn't have the data before that.
512 // Best we can do is to ignore the event.
513 return nil
514 }
515
516 if edit.Editor == nil {
517 return fmt.Errorf("no editor")
518 }
519
520 _, err := b.ResolveTargetWithMetadata(keyGithubId, parseId(edit.Id))
521 if err == nil {
522 // already imported
523 return nil
524 }
525 if err != cache.ErrNoMatchingOp {
526 // real error
527 return err
528 }
529
530 fmt.Println("import edition")
531
532 switch {
533 case edit.DeletedAt != nil:
534 // comment deletion, not supported yet
535
536 case edit.DeletedAt == nil:
537 // comment edition
538 err := b.EditCommentRaw(
539 makePerson(*edit.Editor),
540 edit.CreatedAt.Unix(),
541 target,
542 cleanupText(string(*edit.Diff)),
543 map[string]string{
544 keyGithubId: parseId(edit.Id),
545 },
546 )
547 if err != nil {
548 return err
549 }
550 }
551
552 return nil
553}
554
555// makePerson create a bug.Person from the Github data
556func makePerson(actor actor) bug.Person {
557 return bug.Person{
558 Name: string(actor.Login),
559 AvatarUrl: string(actor.AvatarUrl),
560 }
561}
562
563// parseId convert the unusable githubv4.ID (an interface{}) into a string
564func parseId(id githubv4.ID) string {
565 return fmt.Sprintf("%v", id)
566}
567
568func cleanupText(text string) string {
569 // windows new line, Github, really ?
570 text = strings.Replace(text, "\r\n", "\n", -1)
571
572 // trim extra new line not displayed in the github UI but still present in the data
573 return strings.TrimSpace(text)
574}
575
576func reverseEdits(edits []userContentEdit) []userContentEdit {
577 for i, j := 0, len(edits)-1; i < j; i, j = i+1, j-1 {
578 edits[i], edits[j] = edits[j], edits[i]
579 }
580 return edits
581}