1package github
2
3import (
4 "context"
5 "fmt"
6 "time"
7
8 "github.com/shurcooL/githubv4"
9
10 "github.com/MichaelMure/git-bug/bridge/core"
11 "github.com/MichaelMure/git-bug/bridge/core/auth"
12 "github.com/MichaelMure/git-bug/bug"
13 "github.com/MichaelMure/git-bug/cache"
14 "github.com/MichaelMure/git-bug/entity"
15 "github.com/MichaelMure/git-bug/util/text"
16)
17
18const (
19 metaKeyGithubId = "github-id"
20 metaKeyGithubUrl = "github-url"
21 metaKeyGithubLogin = "github-login"
22)
23
24// githubImporter implement the Importer interface
25type githubImporter struct {
26 conf core.Configuration
27
28 // default user client
29 client *githubv4.Client
30
31 // iterator
32 iterator *iterator
33
34 // send only channel
35 out chan<- core.ImportResult
36}
37
38func (gi *githubImporter) Init(repo *cache.RepoCache, conf core.Configuration) error {
39 gi.conf = conf
40
41 creds, err := auth.List(repo, auth.WithTarget(target), auth.WithKind(auth.KindToken))
42 if err != nil {
43 return err
44 }
45
46 if len(creds) == 0 {
47 return ErrMissingIdentityToken
48 }
49
50 gi.client = buildClient(creds[0].(*auth.Token))
51
52 return nil
53}
54
55// ImportAll iterate over all the configured repository issues and ensure the creation of the
56// missing issues / timeline items / edits / label events ...
57func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
58 gi.iterator = NewIterator(ctx, gi.client, 10, gi.conf[keyOwner], gi.conf[keyProject], since)
59 out := make(chan core.ImportResult)
60 gi.out = out
61
62 go func() {
63 defer close(gi.out)
64
65 // Loop over all matching issues
66 for gi.iterator.NextIssue() {
67 issue := gi.iterator.IssueValue()
68 // create issue
69 b, err := gi.ensureIssue(repo, issue)
70 if err != nil {
71 err := fmt.Errorf("issue creation: %v", err)
72 out <- core.NewImportError(err, "")
73 return
74 }
75
76 // loop over timeline items
77 for gi.iterator.NextTimelineItem() {
78 item := gi.iterator.TimelineItemValue()
79 err := gi.ensureTimelineItem(repo, b, item)
80 if err != nil {
81 err = fmt.Errorf("timeline item creation: %v", err)
82 out <- core.NewImportError(err, "")
83 return
84 }
85 }
86
87 if !b.NeedCommit() {
88 out <- core.NewImportNothing(b.Id(), "no imported operation")
89 } else if err := b.Commit(); err != nil {
90 // commit bug state
91 err = fmt.Errorf("bug commit: %v", err)
92 out <- core.NewImportError(err, "")
93 return
94 }
95 }
96
97 if err := gi.iterator.Error(); err != nil {
98 gi.out <- core.NewImportError(err, "")
99 }
100 }()
101
102 return out, nil
103}
104
105func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline) (*cache.BugCache, error) {
106 // ensure issue author
107 author, err := gi.ensurePerson(repo, issue.Author)
108 if err != nil {
109 return nil, err
110 }
111
112 // resolve bug
113 b, err := repo.ResolveBugCreateMetadata(metaKeyGithubUrl, issue.Url.String())
114 if err != nil && err != bug.ErrBugNotExist {
115 return nil, err
116 }
117
118 // get issue edits
119 var issueEdits []userContentEdit
120 for gi.iterator.NextIssueEdit() {
121 issueEdits = append(issueEdits, gi.iterator.IssueEditValue())
122 }
123
124 // if issueEdits is empty
125 if len(issueEdits) == 0 {
126 if err == bug.ErrBugNotExist {
127 cleanText, err := text.Cleanup(string(issue.Body))
128 if err != nil {
129 return nil, err
130 }
131
132 // create bug
133 b, _, err = repo.NewBugRaw(
134 author,
135 issue.CreatedAt.Unix(),
136 issue.Title,
137 cleanText,
138 nil,
139 map[string]string{
140 core.MetaKeyOrigin: target,
141 metaKeyGithubId: parseId(issue.Id),
142 metaKeyGithubUrl: issue.Url.String(),
143 })
144 if err != nil {
145 return nil, err
146 }
147
148 // importing a new bug
149 gi.out <- core.NewImportBug(b.Id())
150 }
151 } else {
152 // create bug from given issueEdits
153 for i, edit := range issueEdits {
154 if i == 0 && b != nil {
155 // The first edit in the github result is the issue creation itself, we already have that
156 continue
157 }
158
159 cleanText, err := text.Cleanup(string(*edit.Diff))
160 if err != nil {
161 return nil, err
162 }
163
164 // if the bug doesn't exist
165 if b == nil {
166 // we create the bug as soon as we have a legit first edition
167 b, _, err = repo.NewBugRaw(
168 author,
169 issue.CreatedAt.Unix(),
170 issue.Title,
171 cleanText,
172 nil,
173 map[string]string{
174 core.MetaKeyOrigin: target,
175 metaKeyGithubId: parseId(issue.Id),
176 metaKeyGithubUrl: issue.Url.String(),
177 },
178 )
179
180 if err != nil {
181 return nil, err
182 }
183 // importing a new bug
184 gi.out <- core.NewImportBug(b.Id())
185 continue
186 }
187
188 // other edits will be added as CommentEdit operations
189 target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(issue.Id))
190 if err == cache.ErrNoMatchingOp {
191 // original comment is missing somehow, issuing a warning
192 gi.out <- core.NewImportWarning(fmt.Errorf("comment ID %s to edit is missing", parseId(issue.Id)), b.Id())
193 continue
194 }
195 if err != nil {
196 return nil, err
197 }
198
199 err = gi.ensureCommentEdit(repo, b, target, edit)
200 if err != nil {
201 return nil, err
202 }
203 }
204 }
205
206 return b, nil
207}
208
209func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, item timelineItem) error {
210
211 switch item.Typename {
212 case "IssueComment":
213 // collect all comment edits
214 var commentEdits []userContentEdit
215 for gi.iterator.NextCommentEdit() {
216 commentEdits = append(commentEdits, gi.iterator.CommentEditValue())
217 }
218
219 // ensureTimelineComment send import events over out chanel
220 err := gi.ensureTimelineComment(repo, b, item.IssueComment, commentEdits)
221 if err != nil {
222 return fmt.Errorf("timeline comment creation: %v", err)
223 }
224 return nil
225
226 case "LabeledEvent":
227 id := parseId(item.LabeledEvent.Id)
228 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
229 if err == nil {
230 return nil
231 }
232
233 if err != cache.ErrNoMatchingOp {
234 return err
235 }
236 author, err := gi.ensurePerson(repo, item.LabeledEvent.Actor)
237 if err != nil {
238 return err
239 }
240 op, err := b.ForceChangeLabelsRaw(
241 author,
242 item.LabeledEvent.CreatedAt.Unix(),
243 []string{
244 string(item.LabeledEvent.Label.Name),
245 },
246 nil,
247 map[string]string{metaKeyGithubId: id},
248 )
249 if err != nil {
250 return err
251 }
252
253 gi.out <- core.NewImportLabelChange(op.Id())
254 return nil
255
256 case "UnlabeledEvent":
257 id := parseId(item.UnlabeledEvent.Id)
258 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
259 if err == nil {
260 return nil
261 }
262 if err != cache.ErrNoMatchingOp {
263 return err
264 }
265 author, err := gi.ensurePerson(repo, item.UnlabeledEvent.Actor)
266 if err != nil {
267 return err
268 }
269
270 op, err := b.ForceChangeLabelsRaw(
271 author,
272 item.UnlabeledEvent.CreatedAt.Unix(),
273 nil,
274 []string{
275 string(item.UnlabeledEvent.Label.Name),
276 },
277 map[string]string{metaKeyGithubId: id},
278 )
279 if err != nil {
280 return err
281 }
282
283 gi.out <- core.NewImportLabelChange(op.Id())
284 return nil
285
286 case "ClosedEvent":
287 id := parseId(item.ClosedEvent.Id)
288 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
289 if err != cache.ErrNoMatchingOp {
290 return err
291 }
292 if err == nil {
293 return nil
294 }
295 author, err := gi.ensurePerson(repo, item.ClosedEvent.Actor)
296 if err != nil {
297 return err
298 }
299 op, err := b.CloseRaw(
300 author,
301 item.ClosedEvent.CreatedAt.Unix(),
302 map[string]string{metaKeyGithubId: id},
303 )
304
305 if err != nil {
306 return err
307 }
308
309 gi.out <- core.NewImportStatusChange(op.Id())
310 return nil
311
312 case "ReopenedEvent":
313 id := parseId(item.ReopenedEvent.Id)
314 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
315 if err != cache.ErrNoMatchingOp {
316 return err
317 }
318 if err == nil {
319 return nil
320 }
321 author, err := gi.ensurePerson(repo, item.ReopenedEvent.Actor)
322 if err != nil {
323 return err
324 }
325 op, err := b.OpenRaw(
326 author,
327 item.ReopenedEvent.CreatedAt.Unix(),
328 map[string]string{metaKeyGithubId: id},
329 )
330
331 if err != nil {
332 return err
333 }
334
335 gi.out <- core.NewImportStatusChange(op.Id())
336 return nil
337
338 case "RenamedTitleEvent":
339 id := parseId(item.RenamedTitleEvent.Id)
340 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
341 if err != cache.ErrNoMatchingOp {
342 return err
343 }
344 if err == nil {
345 return nil
346 }
347 author, err := gi.ensurePerson(repo, item.RenamedTitleEvent.Actor)
348 if err != nil {
349 return err
350 }
351 op, err := b.SetTitleRaw(
352 author,
353 item.RenamedTitleEvent.CreatedAt.Unix(),
354 string(item.RenamedTitleEvent.CurrentTitle),
355 map[string]string{metaKeyGithubId: id},
356 )
357 if err != nil {
358 return err
359 }
360
361 gi.out <- core.NewImportTitleEdition(op.Id())
362 return nil
363 }
364
365 return nil
366}
367
368func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache.BugCache, item issueComment, edits []userContentEdit) error {
369 // ensure person
370 author, err := gi.ensurePerson(repo, item.Author)
371 if err != nil {
372 return err
373 }
374
375 targetOpID, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(item.Id))
376 if err != nil && err != cache.ErrNoMatchingOp {
377 // real error
378 return err
379 }
380
381 // if no edits are given we create the comment
382 if len(edits) == 0 {
383 if err == cache.ErrNoMatchingOp {
384 cleanText, err := text.Cleanup(string(item.Body))
385 if err != nil {
386 return err
387 }
388
389 // add comment operation
390 op, err := b.AddCommentRaw(
391 author,
392 item.CreatedAt.Unix(),
393 cleanText,
394 nil,
395 map[string]string{
396 metaKeyGithubId: parseId(item.Id),
397 metaKeyGithubUrl: parseId(item.Url.String()),
398 },
399 )
400 if err != nil {
401 return err
402 }
403
404 gi.out <- core.NewImportComment(op.Id())
405 return nil
406 }
407
408 } else {
409 for i, edit := range edits {
410 if i == 0 && targetOpID != "" {
411 // The first edit in the github result is the comment creation itself, we already have that
412 continue
413 }
414
415 // ensure editor identity
416 editor, err := gi.ensurePerson(repo, edit.Editor)
417 if err != nil {
418 return err
419 }
420
421 // create comment when target is empty
422 if targetOpID == "" {
423 cleanText, err := text.Cleanup(string(*edit.Diff))
424 if err != nil {
425 return err
426 }
427
428 op, err := b.AddCommentRaw(
429 editor,
430 edit.CreatedAt.Unix(),
431 cleanText,
432 nil,
433 map[string]string{
434 metaKeyGithubId: parseId(item.Id),
435 metaKeyGithubUrl: item.Url.String(),
436 },
437 )
438 if err != nil {
439 return err
440 }
441 gi.out <- core.NewImportComment(op.Id())
442
443 // set target for the nexr edit now that the comment is created
444 targetOpID = op.Id()
445 continue
446 }
447
448 err = gi.ensureCommentEdit(repo, b, targetOpID, edit)
449 if err != nil {
450 return err
451 }
452 }
453 }
454 return nil
455}
456
457func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugCache, target entity.Id, edit userContentEdit) error {
458 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
459 if err == nil {
460 return nil
461 }
462 if err != cache.ErrNoMatchingOp {
463 // real error
464 return err
465 }
466
467 editor, err := gi.ensurePerson(repo, edit.Editor)
468 if err != nil {
469 return err
470 }
471
472 switch {
473 case edit.DeletedAt != nil:
474 // comment deletion, not supported yet
475 return nil
476
477 case edit.DeletedAt == nil:
478
479 cleanText, err := text.Cleanup(string(*edit.Diff))
480 if err != nil {
481 return err
482 }
483
484 // comment edition
485 op, err := b.EditCommentRaw(
486 editor,
487 edit.CreatedAt.Unix(),
488 target,
489 cleanText,
490 map[string]string{
491 metaKeyGithubId: parseId(edit.Id),
492 },
493 )
494
495 if err != nil {
496 return err
497 }
498
499 gi.out <- core.NewImportCommentEdition(op.Id())
500 return nil
501 }
502 return nil
503}
504
505// ensurePerson create a bug.Person from the Github data
506func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
507 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
508 // in it's UI. So we need a special case to get it.
509 if actor == nil {
510 return gi.getGhost(repo)
511 }
512
513 // Look first in the cache
514 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
515 if err == nil {
516 return i, nil
517 }
518 if entity.IsErrMultipleMatch(err) {
519 return nil, err
520 }
521
522 // importing a new identity
523
524 var name string
525 var email string
526
527 switch actor.Typename {
528 case "User":
529 if actor.User.Name != nil {
530 name = string(*(actor.User.Name))
531 }
532 email = string(actor.User.Email)
533 case "Organization":
534 if actor.Organization.Name != nil {
535 name = string(*(actor.Organization.Name))
536 }
537 if actor.Organization.Email != nil {
538 email = string(*(actor.Organization.Email))
539 }
540 case "Bot":
541 }
542
543 i, err = repo.NewIdentityRaw(
544 name,
545 email,
546 string(actor.AvatarUrl),
547 map[string]string{
548 metaKeyGithubLogin: string(actor.Login),
549 },
550 )
551
552 if err != nil {
553 return nil, err
554 }
555
556 gi.out <- core.NewImportIdentity(i.Id())
557 return i, nil
558}
559
560func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, error) {
561 // Look first in the cache
562 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, "ghost")
563 if err == nil {
564 return i, nil
565 }
566 if entity.IsErrMultipleMatch(err) {
567 return nil, err
568 }
569
570 var q ghostQuery
571
572 variables := map[string]interface{}{
573 "login": githubv4.String("ghost"),
574 }
575
576 ctx, cancel := context.WithTimeout(gi.iterator.ctx, defaultTimeout)
577 defer cancel()
578
579 err = gi.client.Query(ctx, &q, variables)
580 if err != nil {
581 return nil, err
582 }
583
584 var name string
585 if q.User.Name != nil {
586 name = string(*q.User.Name)
587 }
588
589 return repo.NewIdentityRaw(
590 name,
591 "",
592 string(q.User.AvatarUrl),
593 map[string]string{
594 metaKeyGithubLogin: string(q.User.Login),
595 },
596 )
597}
598
599// parseId convert the unusable githubv4.ID (an interface{}) into a string
600func parseId(id githubv4.ID) string {
601 return fmt.Sprintf("%v", id)
602}