1package github
2
3import (
4 "context"
5 "fmt"
6 "time"
7
8 "github.com/shurcooL/githubv4"
9
10 "github.com/MichaelMure/git-bug/bridge/core"
11 "github.com/MichaelMure/git-bug/bridge/core/auth"
12 "github.com/MichaelMure/git-bug/bug"
13 "github.com/MichaelMure/git-bug/cache"
14 "github.com/MichaelMure/git-bug/entity"
15 "github.com/MichaelMure/git-bug/util/text"
16)
17
18// githubImporter implement the Importer interface
19type githubImporter struct {
20 conf core.Configuration
21
22 // default client
23 client *githubv4.Client
24
25 // iterator
26 iterator *iterator
27
28 // send only channel
29 out chan<- core.ImportResult
30}
31
32func (gi *githubImporter) Init(_ context.Context, repo *cache.RepoCache, conf core.Configuration) error {
33 gi.conf = conf
34
35 creds, err := auth.List(repo,
36 auth.WithTarget(target),
37 auth.WithKind(auth.KindToken),
38 auth.WithMeta(auth.MetaKeyLogin, conf[confKeyDefaultLogin]),
39 )
40 if err != nil {
41 return err
42 }
43
44 if len(creds) == 0 {
45 return ErrMissingIdentityToken
46 }
47
48 gi.client = buildClient(creds[0].(*auth.Token))
49
50 return nil
51}
52
53// ImportAll iterate over all the configured repository issues and ensure the creation of the
54// missing issues / timeline items / edits / label events ...
55func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
56 gi.iterator = NewIterator(ctx, gi.client, 10, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
57 out := make(chan core.ImportResult)
58 gi.out = out
59
60 go func() {
61 defer close(gi.out)
62
63 // Loop over all matching issues
64 for gi.iterator.NextIssue() {
65 issue := gi.iterator.IssueValue()
66 // create issue
67 b, err := gi.ensureIssue(repo, issue)
68 if err != nil {
69 err := fmt.Errorf("issue creation: %v", err)
70 out <- core.NewImportError(err, "")
71 return
72 }
73
74 // loop over timeline items
75 for gi.iterator.NextTimelineItem() {
76 item := gi.iterator.TimelineItemValue()
77 err := gi.ensureTimelineItem(repo, b, item)
78 if err != nil {
79 err = fmt.Errorf("timeline item creation: %v", err)
80 out <- core.NewImportError(err, "")
81 return
82 }
83 }
84
85 if !b.NeedCommit() {
86 out <- core.NewImportNothing(b.Id(), "no imported operation")
87 } else if err := b.Commit(); err != nil {
88 // commit bug state
89 err = fmt.Errorf("bug commit: %v", err)
90 out <- core.NewImportError(err, "")
91 return
92 }
93 }
94
95 if err := gi.iterator.Error(); err != nil {
96 gi.out <- core.NewImportError(err, "")
97 }
98 }()
99
100 return out, nil
101}
102
103func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issue) (*cache.BugCache, error) {
104 // ensure issue author
105 author, err := gi.ensurePerson(repo, issue.Author)
106 if err != nil {
107 return nil, err
108 }
109
110 // resolve bug
111 b, err := repo.ResolveBugMatcher(func(excerpt *cache.BugExcerpt) bool {
112 return excerpt.CreateMetadata[core.MetaKeyOrigin] == target &&
113 excerpt.CreateMetadata[metaKeyGithubId] == parseId(issue.Id)
114 })
115 if err != nil && err != bug.ErrBugNotExist {
116 return nil, err
117 }
118
119 // get issue edits
120 var issueEdits []userContentEdit
121 for gi.iterator.NextIssueEdit() {
122 issueEdits = append(issueEdits, gi.iterator.IssueEditValue())
123 }
124
125 // if issueEdits is empty
126 if len(issueEdits) == 0 {
127 if err == bug.ErrBugNotExist {
128 cleanText, err := text.Cleanup(string(issue.Body))
129 if err != nil {
130 return nil, err
131 }
132
133 // create bug
134 b, _, err = repo.NewBugRaw(
135 author,
136 issue.CreatedAt.Unix(),
137 issue.Title,
138 cleanText,
139 nil,
140 map[string]string{
141 core.MetaKeyOrigin: target,
142 metaKeyGithubId: parseId(issue.Id),
143 metaKeyGithubUrl: issue.Url.String(),
144 })
145 if err != nil {
146 return nil, err
147 }
148
149 // importing a new bug
150 gi.out <- core.NewImportBug(b.Id())
151 }
152 } else {
153 // create bug from given issueEdits
154 for i, edit := range issueEdits {
155 if i == 0 && b != nil {
156 // The first edit in the github result is the issue creation itself, we already have that
157 continue
158 }
159
160 cleanText, err := text.Cleanup(string(*edit.Diff))
161 if err != nil {
162 return nil, err
163 }
164
165 // if the bug doesn't exist
166 if b == nil {
167 // we create the bug as soon as we have a legit first edition
168 b, _, err = repo.NewBugRaw(
169 author,
170 issue.CreatedAt.Unix(),
171 issue.Title, // TODO: this is the *current* title, not the original one
172 cleanText,
173 nil,
174 map[string]string{
175 core.MetaKeyOrigin: target,
176 metaKeyGithubId: parseId(issue.Id),
177 metaKeyGithubUrl: issue.Url.String(),
178 },
179 )
180
181 if err != nil {
182 return nil, err
183 }
184 // importing a new bug
185 gi.out <- core.NewImportBug(b.Id())
186 continue
187 }
188
189 // other edits will be added as CommentEdit operations
190 target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(issue.Id))
191 if err == cache.ErrNoMatchingOp {
192 // original comment is missing somehow, issuing a warning
193 gi.out <- core.NewImportWarning(fmt.Errorf("comment ID %s to edit is missing", parseId(issue.Id)), b.Id())
194 continue
195 }
196 if err != nil {
197 return nil, err
198 }
199
200 err = gi.ensureCommentEdit(repo, b, target, edit)
201 if err != nil {
202 return nil, err
203 }
204 }
205 }
206
207 return b, nil
208}
209
210func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, item timelineItem) error {
211
212 switch item.Typename {
213 case "IssueComment":
214 // collect all comment edits
215 var commentEdits []userContentEdit
216 for gi.iterator.NextCommentEdit() {
217 commentEdits = append(commentEdits, gi.iterator.CommentEditValue())
218 }
219
220 // ensureTimelineComment send import events over out chanel
221 err := gi.ensureTimelineComment(repo, b, item.IssueComment, commentEdits)
222 if err != nil {
223 return fmt.Errorf("timeline comment creation: %v", err)
224 }
225 return nil
226
227 case "LabeledEvent":
228 id := parseId(item.LabeledEvent.Id)
229 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
230 if err == nil {
231 return nil
232 }
233
234 if err != cache.ErrNoMatchingOp {
235 return err
236 }
237 author, err := gi.ensurePerson(repo, item.LabeledEvent.Actor)
238 if err != nil {
239 return err
240 }
241 op, err := b.ForceChangeLabelsRaw(
242 author,
243 item.LabeledEvent.CreatedAt.Unix(),
244 []string{
245 string(item.LabeledEvent.Label.Name),
246 },
247 nil,
248 map[string]string{metaKeyGithubId: id},
249 )
250 if err != nil {
251 return err
252 }
253
254 gi.out <- core.NewImportLabelChange(op.Id())
255 return nil
256
257 case "UnlabeledEvent":
258 id := parseId(item.UnlabeledEvent.Id)
259 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
260 if err == nil {
261 return nil
262 }
263 if err != cache.ErrNoMatchingOp {
264 return err
265 }
266 author, err := gi.ensurePerson(repo, item.UnlabeledEvent.Actor)
267 if err != nil {
268 return err
269 }
270
271 op, err := b.ForceChangeLabelsRaw(
272 author,
273 item.UnlabeledEvent.CreatedAt.Unix(),
274 nil,
275 []string{
276 string(item.UnlabeledEvent.Label.Name),
277 },
278 map[string]string{metaKeyGithubId: id},
279 )
280 if err != nil {
281 return err
282 }
283
284 gi.out <- core.NewImportLabelChange(op.Id())
285 return nil
286
287 case "ClosedEvent":
288 id := parseId(item.ClosedEvent.Id)
289 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
290 if err != cache.ErrNoMatchingOp {
291 return err
292 }
293 if err == nil {
294 return nil
295 }
296 author, err := gi.ensurePerson(repo, item.ClosedEvent.Actor)
297 if err != nil {
298 return err
299 }
300 op, err := b.CloseRaw(
301 author,
302 item.ClosedEvent.CreatedAt.Unix(),
303 map[string]string{metaKeyGithubId: id},
304 )
305
306 if err != nil {
307 return err
308 }
309
310 gi.out <- core.NewImportStatusChange(op.Id())
311 return nil
312
313 case "ReopenedEvent":
314 id := parseId(item.ReopenedEvent.Id)
315 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
316 if err != cache.ErrNoMatchingOp {
317 return err
318 }
319 if err == nil {
320 return nil
321 }
322 author, err := gi.ensurePerson(repo, item.ReopenedEvent.Actor)
323 if err != nil {
324 return err
325 }
326 op, err := b.OpenRaw(
327 author,
328 item.ReopenedEvent.CreatedAt.Unix(),
329 map[string]string{metaKeyGithubId: id},
330 )
331
332 if err != nil {
333 return err
334 }
335
336 gi.out <- core.NewImportStatusChange(op.Id())
337 return nil
338
339 case "RenamedTitleEvent":
340 id := parseId(item.RenamedTitleEvent.Id)
341 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
342 if err != cache.ErrNoMatchingOp {
343 return err
344 }
345 if err == nil {
346 return nil
347 }
348 author, err := gi.ensurePerson(repo, item.RenamedTitleEvent.Actor)
349 if err != nil {
350 return err
351 }
352 op, err := b.SetTitleRaw(
353 author,
354 item.RenamedTitleEvent.CreatedAt.Unix(),
355 string(item.RenamedTitleEvent.CurrentTitle),
356 map[string]string{metaKeyGithubId: id},
357 )
358 if err != nil {
359 return err
360 }
361
362 gi.out <- core.NewImportTitleEdition(op.Id())
363 return nil
364 }
365
366 return nil
367}
368
369func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache.BugCache, item issueComment, edits []userContentEdit) error {
370 // ensure person
371 author, err := gi.ensurePerson(repo, item.Author)
372 if err != nil {
373 return err
374 }
375
376 targetOpID, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(item.Id))
377 if err != nil && err != cache.ErrNoMatchingOp {
378 // real error
379 return err
380 }
381
382 // if no edits are given we create the comment
383 if len(edits) == 0 {
384 if err == cache.ErrNoMatchingOp {
385 cleanText, err := text.Cleanup(string(item.Body))
386 if err != nil {
387 return err
388 }
389
390 // add comment operation
391 op, err := b.AddCommentRaw(
392 author,
393 item.CreatedAt.Unix(),
394 cleanText,
395 nil,
396 map[string]string{
397 metaKeyGithubId: parseId(item.Id),
398 metaKeyGithubUrl: parseId(item.Url.String()),
399 },
400 )
401 if err != nil {
402 return err
403 }
404
405 gi.out <- core.NewImportComment(op.Id())
406 return nil
407 }
408
409 } else {
410 for i, edit := range edits {
411 if i == 0 && targetOpID != "" {
412 // The first edit in the github result is the comment creation itself, we already have that
413 continue
414 }
415
416 // ensure editor identity
417 editor, err := gi.ensurePerson(repo, edit.Editor)
418 if err != nil {
419 return err
420 }
421
422 // create comment when target is empty
423 if targetOpID == "" {
424 cleanText, err := text.Cleanup(string(*edit.Diff))
425 if err != nil {
426 return err
427 }
428
429 op, err := b.AddCommentRaw(
430 editor,
431 edit.CreatedAt.Unix(),
432 cleanText,
433 nil,
434 map[string]string{
435 metaKeyGithubId: parseId(item.Id),
436 metaKeyGithubUrl: item.Url.String(),
437 },
438 )
439 if err != nil {
440 return err
441 }
442 gi.out <- core.NewImportComment(op.Id())
443
444 // set target for the next edit now that the comment is created
445 targetOpID = op.Id()
446 continue
447 }
448
449 err = gi.ensureCommentEdit(repo, b, targetOpID, edit)
450 if err != nil {
451 return err
452 }
453 }
454 }
455 return nil
456}
457
458func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugCache, target entity.Id, edit userContentEdit) error {
459 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
460 if err == nil {
461 return nil
462 }
463 if err != cache.ErrNoMatchingOp {
464 // real error
465 return err
466 }
467
468 editor, err := gi.ensurePerson(repo, edit.Editor)
469 if err != nil {
470 return err
471 }
472
473 switch {
474 case edit.DeletedAt != nil:
475 // comment deletion, not supported yet
476 return nil
477
478 case edit.DeletedAt == nil:
479
480 cleanText, err := text.Cleanup(string(*edit.Diff))
481 if err != nil {
482 return err
483 }
484
485 // comment edition
486 op, err := b.EditCommentRaw(
487 editor,
488 edit.CreatedAt.Unix(),
489 target,
490 cleanText,
491 map[string]string{
492 metaKeyGithubId: parseId(edit.Id),
493 },
494 )
495
496 if err != nil {
497 return err
498 }
499
500 gi.out <- core.NewImportCommentEdition(op.Id())
501 return nil
502 }
503 return nil
504}
505
506// ensurePerson create a bug.Person from the Github data
507func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
508 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
509 // in it's UI. So we need a special case to get it.
510 if actor == nil {
511 return gi.getGhost(repo)
512 }
513
514 // Look first in the cache
515 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
516 if err == nil {
517 return i, nil
518 }
519 if entity.IsErrMultipleMatch(err) {
520 return nil, err
521 }
522
523 // importing a new identity
524
525 var name string
526 var email string
527
528 switch actor.Typename {
529 case "User":
530 if actor.User.Name != nil {
531 name = string(*(actor.User.Name))
532 }
533 email = string(actor.User.Email)
534 case "Organization":
535 if actor.Organization.Name != nil {
536 name = string(*(actor.Organization.Name))
537 }
538 if actor.Organization.Email != nil {
539 email = string(*(actor.Organization.Email))
540 }
541 case "Bot":
542 }
543
544 // Name is not necessarily set, fallback to login as a name is required in the identity
545 if name == "" {
546 name = string(actor.Login)
547 }
548
549 i, err = repo.NewIdentityRaw(
550 name,
551 email,
552 string(actor.Login),
553 string(actor.AvatarUrl),
554 map[string]string{
555 metaKeyGithubLogin: string(actor.Login),
556 },
557 )
558
559 if err != nil {
560 return nil, err
561 }
562
563 gi.out <- core.NewImportIdentity(i.Id())
564 return i, nil
565}
566
567func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, error) {
568 // Look first in the cache
569 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, "ghost")
570 if err == nil {
571 return i, nil
572 }
573 if entity.IsErrMultipleMatch(err) {
574 return nil, err
575 }
576
577 var q ghostQuery
578
579 variables := map[string]interface{}{
580 "login": githubv4.String("ghost"),
581 }
582
583 ctx, cancel := context.WithTimeout(gi.iterator.ctx, defaultTimeout)
584 defer cancel()
585
586 err = gi.client.Query(ctx, &q, variables)
587 if err != nil {
588 return nil, err
589 }
590
591 var name string
592 if q.User.Name != nil {
593 name = string(*q.User.Name)
594 }
595
596 return repo.NewIdentityRaw(
597 name,
598 "",
599 string(q.User.Login),
600 string(q.User.AvatarUrl),
601 map[string]string{
602 metaKeyGithubLogin: string(q.User.Login),
603 },
604 )
605}
606
607// parseId convert the unusable githubv4.ID (an interface{}) into a string
608func parseId(id githubv4.ID) string {
609 return fmt.Sprintf("%v", id)
610}