1package github
2
3import (
4 "context"
5 "fmt"
6 "time"
7
8 "github.com/shurcooL/githubv4"
9
10 "github.com/MichaelMure/git-bug/bridge/core"
11 "github.com/MichaelMure/git-bug/bug"
12 "github.com/MichaelMure/git-bug/cache"
13 "github.com/MichaelMure/git-bug/entity"
14 "github.com/MichaelMure/git-bug/util/text"
15)
16
17const (
18 metaKeyGithubId = "github-id"
19 metaKeyGithubUrl = "github-url"
20 metaKeyGithubLogin = "github-login"
21)
22
23// githubImporter implement the Importer interface
24type githubImporter struct {
25 conf core.Configuration
26
27 // iterator
28 iterator *iterator
29
30 // send only channel
31 out chan<- core.ImportResult
32}
33
34func (gi *githubImporter) Init(conf core.Configuration) error {
35 gi.conf = conf
36 return nil
37}
38
39// ImportAll iterate over all the configured repository issues and ensure the creation of the
40// missing issues / timeline items / edits / label events ...
41func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
42 gi.iterator = NewIterator(ctx, 10, gi.conf[keyOwner], gi.conf[keyProject], gi.conf[keyToken], since)
43 out := make(chan core.ImportResult)
44 gi.out = out
45
46 go func() {
47 defer close(gi.out)
48
49 // Loop over all matching issues
50 for gi.iterator.NextIssue() {
51 issue := gi.iterator.IssueValue()
52 // create issue
53 b, err := gi.ensureIssue(repo, issue)
54 if err != nil {
55 err := fmt.Errorf("issue creation: %v", err)
56 out <- core.NewImportError(err, "")
57 return
58 }
59
60 // loop over timeline items
61 for gi.iterator.NextTimelineItem() {
62 item := gi.iterator.TimelineItemValue()
63 if err := gi.ensureTimelineItem(repo, b, item); err != nil {
64 err := fmt.Errorf("timeline item creation: %v", err)
65 out <- core.NewImportError(err, "")
66 return
67 }
68 }
69
70 // commit bug state
71 if err := b.CommitAsNeeded(); err != nil {
72 err = fmt.Errorf("bug commit: %v", err)
73 out <- core.NewImportError(err, "")
74 return
75 }
76 }
77
78 if err := gi.iterator.Error(); err != nil && err != context.Canceled {
79 gi.out <- core.NewImportError(err, "")
80 }
81 }()
82
83 return out, nil
84}
85
86func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline) (*cache.BugCache, error) {
87 // ensure issue author
88 author, err := gi.ensurePerson(repo, issue.Author)
89 if err != nil {
90 return nil, err
91 }
92
93 // resolve bug
94 b, err := repo.ResolveBugCreateMetadata(metaKeyGithubUrl, issue.Url.String())
95 if err != nil && err != bug.ErrBugNotExist {
96 return nil, err
97 }
98
99 // get issue edits
100 var issueEdits []userContentEdit
101 for gi.iterator.NextIssueEdit() {
102 issueEdits = append(issueEdits, gi.iterator.IssueEditValue())
103 }
104
105 // if issueEdits is empty
106 if len(issueEdits) == 0 {
107 if err == bug.ErrBugNotExist {
108 cleanText, err := text.Cleanup(string(issue.Body))
109 if err != nil {
110 return nil, err
111 }
112
113 // create bug
114 b, _, err = repo.NewBugRaw(
115 author,
116 issue.CreatedAt.Unix(),
117 issue.Title,
118 cleanText,
119 nil,
120 map[string]string{
121 core.MetaKeyOrigin: target,
122 metaKeyGithubId: parseId(issue.Id),
123 metaKeyGithubUrl: issue.Url.String(),
124 })
125 if err != nil {
126 return nil, err
127 }
128
129 // importing a new bug
130 gi.out <- core.NewImportBug(b.Id())
131 } else {
132 gi.out <- core.NewImportNothing("", "bug already imported")
133 }
134
135 } else {
136 // create bug from given issueEdits
137 for i, edit := range issueEdits {
138 if i == 0 && b != nil {
139 // The first edit in the github result is the issue creation itself, we already have that
140 gi.out <- core.NewImportNothing("", "bug already imported")
141 continue
142 }
143
144 cleanText, err := text.Cleanup(string(*edit.Diff))
145 if err != nil {
146 return nil, err
147 }
148
149 // if the bug doesn't exist
150 if b == nil {
151 // we create the bug as soon as we have a legit first edition
152 b, _, err = repo.NewBugRaw(
153 author,
154 issue.CreatedAt.Unix(),
155 issue.Title,
156 cleanText,
157 nil,
158 map[string]string{
159 core.MetaKeyOrigin: target,
160 metaKeyGithubId: parseId(issue.Id),
161 metaKeyGithubUrl: issue.Url.String(),
162 },
163 )
164
165 if err != nil {
166 return nil, err
167 }
168
169 // importing a new bug
170 gi.out <- core.NewImportBug(b.Id())
171 continue
172 }
173
174 // other edits will be added as CommentEdit operations
175 target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(issue.Id))
176 if err != nil {
177 return nil, err
178 }
179
180 err = gi.ensureCommentEdit(repo, b, target, edit)
181 if err != nil {
182 return nil, err
183 }
184 }
185 }
186
187 return b, nil
188}
189
190func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, item timelineItem) error {
191
192 switch item.Typename {
193 case "IssueComment":
194 // collect all comment edits
195 var commentEdits []userContentEdit
196 for gi.iterator.NextCommentEdit() {
197 commentEdits = append(commentEdits, gi.iterator.CommentEditValue())
198 }
199
200 // ensureTimelineComment send import events over out chanel
201 err := gi.ensureTimelineComment(repo, b, item.IssueComment, commentEdits)
202 if err != nil {
203 return fmt.Errorf("timeline comment creation: %v", err)
204 }
205
206 case "LabeledEvent":
207 id := parseId(item.LabeledEvent.Id)
208 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
209 if err == nil {
210 reason := fmt.Sprintf("operation already imported: %v", item.Typename)
211 gi.out <- core.NewImportNothing("", reason)
212 return nil
213 }
214
215 if err != cache.ErrNoMatchingOp {
216 return err
217 }
218 author, err := gi.ensurePerson(repo, item.LabeledEvent.Actor)
219 if err != nil {
220 return err
221 }
222 op, err := b.ForceChangeLabelsRaw(
223 author,
224 item.LabeledEvent.CreatedAt.Unix(),
225 []string{
226 string(item.LabeledEvent.Label.Name),
227 },
228 nil,
229 map[string]string{metaKeyGithubId: id},
230 )
231 if err != nil {
232 return err
233 }
234
235 gi.out <- core.NewImportLabelChange(op.Id())
236 return nil
237
238 case "UnlabeledEvent":
239 id := parseId(item.UnlabeledEvent.Id)
240 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
241 if err == nil {
242 reason := fmt.Sprintf("operation already imported: %v", item.Typename)
243 gi.out <- core.NewImportNothing("", reason)
244 return nil
245 }
246 if err != cache.ErrNoMatchingOp {
247 return err
248 }
249 author, err := gi.ensurePerson(repo, item.UnlabeledEvent.Actor)
250 if err != nil {
251 return err
252 }
253
254 op, err := b.ForceChangeLabelsRaw(
255 author,
256 item.UnlabeledEvent.CreatedAt.Unix(),
257 nil,
258 []string{
259 string(item.UnlabeledEvent.Label.Name),
260 },
261 map[string]string{metaKeyGithubId: id},
262 )
263 if err != nil {
264 return err
265 }
266
267 gi.out <- core.NewImportLabelChange(op.Id())
268 return nil
269
270 case "ClosedEvent":
271 id := parseId(item.ClosedEvent.Id)
272 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
273 if err != cache.ErrNoMatchingOp {
274 return err
275 }
276 if err == nil {
277 reason := fmt.Sprintf("operation already imported: %v", item.Typename)
278 gi.out <- core.NewImportNothing("", reason)
279 return nil
280 }
281 author, err := gi.ensurePerson(repo, item.ClosedEvent.Actor)
282 if err != nil {
283 return err
284 }
285 op, err := b.CloseRaw(
286 author,
287 item.ClosedEvent.CreatedAt.Unix(),
288 map[string]string{metaKeyGithubId: id},
289 )
290
291 if err != nil {
292 return err
293 }
294
295 gi.out <- core.NewImportStatusChange(op.Id())
296 return nil
297
298 case "ReopenedEvent":
299 id := parseId(item.ReopenedEvent.Id)
300 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
301 if err != cache.ErrNoMatchingOp {
302 return err
303 }
304 if err == nil {
305 reason := fmt.Sprintf("operation already imported: %v", item.Typename)
306 gi.out <- core.NewImportNothing("", reason)
307 return nil
308 }
309 author, err := gi.ensurePerson(repo, item.ReopenedEvent.Actor)
310 if err != nil {
311 return err
312 }
313 op, err := b.OpenRaw(
314 author,
315 item.ReopenedEvent.CreatedAt.Unix(),
316 map[string]string{metaKeyGithubId: id},
317 )
318
319 if err != nil {
320 return err
321 }
322
323 gi.out <- core.NewImportStatusChange(op.Id())
324 return nil
325
326 case "RenamedTitleEvent":
327 id := parseId(item.RenamedTitleEvent.Id)
328 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
329 if err != cache.ErrNoMatchingOp {
330 return err
331 }
332 if err == nil {
333 reason := fmt.Sprintf("operation already imported: %v", item.Typename)
334 gi.out <- core.NewImportNothing("", reason)
335 return nil
336 }
337 author, err := gi.ensurePerson(repo, item.RenamedTitleEvent.Actor)
338 if err != nil {
339 return err
340 }
341 op, err := b.SetTitleRaw(
342 author,
343 item.RenamedTitleEvent.CreatedAt.Unix(),
344 string(item.RenamedTitleEvent.CurrentTitle),
345 map[string]string{metaKeyGithubId: id},
346 )
347 if err != nil {
348 return err
349 }
350
351 gi.out <- core.NewImportTitleEdition(op.Id())
352 return nil
353
354 default:
355 reason := fmt.Sprintf("ignoring timeline type: %v", item.Typename)
356 gi.out <- core.NewImportNothing("", reason)
357 }
358
359 return nil
360}
361
362func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache.BugCache, item issueComment, edits []userContentEdit) error {
363 // ensure person
364 author, err := gi.ensurePerson(repo, item.Author)
365 if err != nil {
366 return err
367 }
368
369 targetOpID, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(item.Id))
370 if err == nil {
371 gi.out <- core.NewImportNothing("", "comment already imported")
372 } else if err != cache.ErrNoMatchingOp {
373 // real error
374 return err
375 }
376
377 // if no edits are given we create the comment
378 if len(edits) == 0 {
379 if err == cache.ErrNoMatchingOp {
380 cleanText, err := text.Cleanup(string(item.Body))
381 if err != nil {
382 return err
383 }
384
385 // add comment operation
386 op, err := b.AddCommentRaw(
387 author,
388 item.CreatedAt.Unix(),
389 cleanText,
390 nil,
391 map[string]string{
392 metaKeyGithubId: parseId(item.Id),
393 metaKeyGithubUrl: parseId(item.Url.String()),
394 },
395 )
396 if err != nil {
397 return err
398 }
399
400 gi.out <- core.NewImportComment(op.Id())
401 }
402
403 } else {
404 for i, edit := range edits {
405 if i == 0 && targetOpID != "" {
406 // The first edit in the github result is the comment creation itself, we already have that
407 gi.out <- core.NewImportNothing("", "comment already imported")
408 continue
409 }
410
411 // ensure editor identity
412 editor, err := gi.ensurePerson(repo, edit.Editor)
413 if err != nil {
414 return err
415 }
416
417 // create comment when target is empty
418 if targetOpID == "" {
419 cleanText, err := text.Cleanup(string(*edit.Diff))
420 if err != nil {
421 return err
422 }
423
424 op, err := b.AddCommentRaw(
425 editor,
426 edit.CreatedAt.Unix(),
427 cleanText,
428 nil,
429 map[string]string{
430 metaKeyGithubId: parseId(item.Id),
431 metaKeyGithubUrl: item.Url.String(),
432 },
433 )
434 if err != nil {
435 return err
436 }
437
438 // set target for the nexr edit now that the comment is created
439 targetOpID = op.Id()
440 continue
441 }
442
443 err = gi.ensureCommentEdit(repo, b, targetOpID, edit)
444 if err != nil {
445 return err
446 }
447 }
448 }
449 return nil
450}
451
452func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugCache, target entity.Id, edit userContentEdit) error {
453 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
454 if err == nil {
455 gi.out <- core.NewImportNothing(b.Id(), "edition already imported")
456 return nil
457 }
458 if err != cache.ErrNoMatchingOp {
459 // real error
460 return err
461 }
462
463 editor, err := gi.ensurePerson(repo, edit.Editor)
464 if err != nil {
465 return err
466 }
467
468 switch {
469 case edit.DeletedAt != nil:
470 // comment deletion, not supported yet
471 gi.out <- core.NewImportNothing(b.Id(), "comment deletion is not supported yet")
472
473 case edit.DeletedAt == nil:
474
475 cleanText, err := text.Cleanup(string(*edit.Diff))
476 if err != nil {
477 return err
478 }
479
480 // comment edition
481 op, err := b.EditCommentRaw(
482 editor,
483 edit.CreatedAt.Unix(),
484 target,
485 cleanText,
486 map[string]string{
487 metaKeyGithubId: parseId(edit.Id),
488 },
489 )
490
491 if err != nil {
492 return err
493 }
494
495 gi.out <- core.NewImportCommentEdition(op.Id())
496 }
497
498 return nil
499}
500
501// ensurePerson create a bug.Person from the Github data
502func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
503 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
504 // in it's UI. So we need a special case to get it.
505 if actor == nil {
506 return gi.getGhost(repo)
507 }
508
509 // Look first in the cache
510 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
511 if err == nil {
512 return i, nil
513 }
514 if _, ok := err.(entity.ErrMultipleMatch); ok {
515 return nil, err
516 }
517
518 // importing a new identity
519
520 var name string
521 var email string
522
523 switch actor.Typename {
524 case "User":
525 if actor.User.Name != nil {
526 name = string(*(actor.User.Name))
527 }
528 email = string(actor.User.Email)
529 case "Organization":
530 if actor.Organization.Name != nil {
531 name = string(*(actor.Organization.Name))
532 }
533 if actor.Organization.Email != nil {
534 email = string(*(actor.Organization.Email))
535 }
536 case "Bot":
537 }
538
539 i, err = repo.NewIdentityRaw(
540 name,
541 email,
542 string(actor.Login),
543 string(actor.AvatarUrl),
544 map[string]string{
545 metaKeyGithubLogin: string(actor.Login),
546 },
547 )
548
549 if err != nil {
550 return nil, err
551 }
552
553 gi.out <- core.NewImportIdentity(i.Id())
554 return i, nil
555}
556
557func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, error) {
558 // Look first in the cache
559 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, "ghost")
560 if err == nil {
561 return i, nil
562 }
563 if _, ok := err.(entity.ErrMultipleMatch); ok {
564 return nil, err
565 }
566
567 var q ghostQuery
568
569 variables := map[string]interface{}{
570 "login": githubv4.String("ghost"),
571 }
572
573 gc := buildClient(gi.conf[keyToken])
574
575 ctx, cancel := context.WithTimeout(gi.iterator.ctx, defaultTimeout)
576 defer cancel()
577
578 err = gc.Query(ctx, &q, variables)
579 if err != nil {
580 return nil, err
581 }
582
583 var name string
584 if q.User.Name != nil {
585 name = string(*q.User.Name)
586 }
587
588 return repo.NewIdentityRaw(
589 name,
590 "",
591 string(q.User.Login),
592 string(q.User.AvatarUrl),
593 map[string]string{
594 metaKeyGithubLogin: string(q.User.Login),
595 },
596 )
597}
598
599// parseId convert the unusable githubv4.ID (an interface{}) into a string
600func parseId(id githubv4.ID) string {
601 return fmt.Sprintf("%v", id)
602}