1package github
2
3import (
4 "context"
5 "fmt"
6 "time"
7
8 "github.com/shurcooL/githubv4"
9
10 "github.com/MichaelMure/git-bug/bridge/core"
11 "github.com/MichaelMure/git-bug/bridge/core/auth"
12 "github.com/MichaelMure/git-bug/bug"
13 "github.com/MichaelMure/git-bug/cache"
14 "github.com/MichaelMure/git-bug/entity"
15 "github.com/MichaelMure/git-bug/util/text"
16)
17
18const (
19 metaKeyGithubId = "github-id"
20 metaKeyGithubUrl = "github-url"
21 metaKeyGithubLogin = "github-login"
22)
23
24// githubImporter implement the Importer interface
25type githubImporter struct {
26 conf core.Configuration
27
28 // default user client
29 client *githubv4.Client
30
31 // iterator
32 iterator *iterator
33
34 // send only channel
35 out chan<- core.ImportResult
36}
37
38func (gi *githubImporter) Init(repo *cache.RepoCache, conf core.Configuration) error {
39 gi.conf = conf
40
41 opts := []auth.Option{
42 auth.WithTarget(target),
43 auth.WithKind(auth.KindToken),
44 }
45
46 user, err := repo.GetUserIdentity()
47 if err == nil {
48 opts = append(opts, auth.WithUserId(user.Id()))
49 }
50
51 creds, err := auth.List(repo, opts...)
52 if err != nil {
53 return err
54 }
55
56 if len(creds) == 0 {
57 return ErrMissingIdentityToken
58 }
59
60 gi.client = buildClient(creds[0].(*auth.Token))
61
62 return nil
63}
64
65// ImportAll iterate over all the configured repository issues and ensure the creation of the
66// missing issues / timeline items / edits / label events ...
67func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
68 gi.iterator = NewIterator(ctx, gi.client, 10, gi.conf[keyOwner], gi.conf[keyProject], since)
69 out := make(chan core.ImportResult)
70 gi.out = out
71
72 go func() {
73 defer close(gi.out)
74
75 // Loop over all matching issues
76 for gi.iterator.NextIssue() {
77 issue := gi.iterator.IssueValue()
78 // create issue
79 b, err := gi.ensureIssue(repo, issue)
80 if err != nil {
81 err := fmt.Errorf("issue creation: %v", err)
82 out <- core.NewImportError(err, "")
83 return
84 }
85
86 // loop over timeline items
87 for gi.iterator.NextTimelineItem() {
88 item := gi.iterator.TimelineItemValue()
89 err := gi.ensureTimelineItem(repo, b, item)
90 if err != nil {
91 err = fmt.Errorf("timeline item creation: %v", err)
92 out <- core.NewImportError(err, "")
93 return
94 }
95 }
96
97 if !b.NeedCommit() {
98 out <- core.NewImportNothing(b.Id(), "no imported operation")
99 } else if err := b.Commit(); err != nil {
100 // commit bug state
101 err = fmt.Errorf("bug commit: %v", err)
102 out <- core.NewImportError(err, "")
103 return
104 }
105 }
106
107 if err := gi.iterator.Error(); err != nil && err != context.Canceled {
108 gi.out <- core.NewImportError(err, "")
109 }
110 }()
111
112 return out, nil
113}
114
115func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline) (*cache.BugCache, error) {
116 // ensure issue author
117 author, err := gi.ensurePerson(repo, issue.Author)
118 if err != nil {
119 return nil, err
120 }
121
122 // resolve bug
123 b, err := repo.ResolveBugCreateMetadata(metaKeyGithubUrl, issue.Url.String())
124 if err != nil && err != bug.ErrBugNotExist {
125 return nil, err
126 }
127
128 // get issue edits
129 var issueEdits []userContentEdit
130 for gi.iterator.NextIssueEdit() {
131 issueEdits = append(issueEdits, gi.iterator.IssueEditValue())
132 }
133
134 // if issueEdits is empty
135 if len(issueEdits) == 0 {
136 if err == bug.ErrBugNotExist {
137 cleanText, err := text.Cleanup(string(issue.Body))
138 if err != nil {
139 return nil, err
140 }
141
142 // create bug
143 b, _, err = repo.NewBugRaw(
144 author,
145 issue.CreatedAt.Unix(),
146 issue.Title,
147 cleanText,
148 nil,
149 map[string]string{
150 core.MetaKeyOrigin: target,
151 metaKeyGithubId: parseId(issue.Id),
152 metaKeyGithubUrl: issue.Url.String(),
153 })
154 if err != nil {
155 return nil, err
156 }
157
158 // importing a new bug
159 gi.out <- core.NewImportBug(b.Id())
160 }
161 } else {
162 // create bug from given issueEdits
163 for i, edit := range issueEdits {
164 if i == 0 && b != nil {
165 // The first edit in the github result is the issue creation itself, we already have that
166 continue
167 }
168
169 cleanText, err := text.Cleanup(string(*edit.Diff))
170 if err != nil {
171 return nil, err
172 }
173
174 // if the bug doesn't exist
175 if b == nil {
176 // we create the bug as soon as we have a legit first edition
177 b, _, err = repo.NewBugRaw(
178 author,
179 issue.CreatedAt.Unix(),
180 issue.Title,
181 cleanText,
182 nil,
183 map[string]string{
184 core.MetaKeyOrigin: target,
185 metaKeyGithubId: parseId(issue.Id),
186 metaKeyGithubUrl: issue.Url.String(),
187 },
188 )
189
190 if err != nil {
191 return nil, err
192 }
193 // importing a new bug
194 gi.out <- core.NewImportBug(b.Id())
195 continue
196 }
197
198 // other edits will be added as CommentEdit operations
199 target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(issue.Id))
200 if err != nil {
201 return nil, err
202 }
203
204 err = gi.ensureCommentEdit(repo, b, target, edit)
205 if err != nil {
206 return nil, err
207 }
208 }
209 }
210
211 return b, nil
212}
213
214func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, item timelineItem) error {
215
216 switch item.Typename {
217 case "IssueComment":
218 // collect all comment edits
219 var commentEdits []userContentEdit
220 for gi.iterator.NextCommentEdit() {
221 commentEdits = append(commentEdits, gi.iterator.CommentEditValue())
222 }
223
224 // ensureTimelineComment send import events over out chanel
225 err := gi.ensureTimelineComment(repo, b, item.IssueComment, commentEdits)
226 if err != nil {
227 return fmt.Errorf("timeline comment creation: %v", err)
228 }
229 return nil
230
231 case "LabeledEvent":
232 id := parseId(item.LabeledEvent.Id)
233 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
234 if err == nil {
235 return nil
236 }
237
238 if err != cache.ErrNoMatchingOp {
239 return err
240 }
241 author, err := gi.ensurePerson(repo, item.LabeledEvent.Actor)
242 if err != nil {
243 return err
244 }
245 op, err := b.ForceChangeLabelsRaw(
246 author,
247 item.LabeledEvent.CreatedAt.Unix(),
248 []string{
249 string(item.LabeledEvent.Label.Name),
250 },
251 nil,
252 map[string]string{metaKeyGithubId: id},
253 )
254 if err != nil {
255 return err
256 }
257
258 gi.out <- core.NewImportLabelChange(op.Id())
259 return nil
260
261 case "UnlabeledEvent":
262 id := parseId(item.UnlabeledEvent.Id)
263 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
264 if err == nil {
265 return nil
266 }
267 if err != cache.ErrNoMatchingOp {
268 return err
269 }
270 author, err := gi.ensurePerson(repo, item.UnlabeledEvent.Actor)
271 if err != nil {
272 return err
273 }
274
275 op, err := b.ForceChangeLabelsRaw(
276 author,
277 item.UnlabeledEvent.CreatedAt.Unix(),
278 nil,
279 []string{
280 string(item.UnlabeledEvent.Label.Name),
281 },
282 map[string]string{metaKeyGithubId: id},
283 )
284 if err != nil {
285 return err
286 }
287
288 gi.out <- core.NewImportLabelChange(op.Id())
289 return nil
290
291 case "ClosedEvent":
292 id := parseId(item.ClosedEvent.Id)
293 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
294 if err != cache.ErrNoMatchingOp {
295 return err
296 }
297 if err == nil {
298 return nil
299 }
300 author, err := gi.ensurePerson(repo, item.ClosedEvent.Actor)
301 if err != nil {
302 return err
303 }
304 op, err := b.CloseRaw(
305 author,
306 item.ClosedEvent.CreatedAt.Unix(),
307 map[string]string{metaKeyGithubId: id},
308 )
309
310 if err != nil {
311 return err
312 }
313
314 gi.out <- core.NewImportStatusChange(op.Id())
315 return nil
316
317 case "ReopenedEvent":
318 id := parseId(item.ReopenedEvent.Id)
319 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
320 if err != cache.ErrNoMatchingOp {
321 return err
322 }
323 if err == nil {
324 return nil
325 }
326 author, err := gi.ensurePerson(repo, item.ReopenedEvent.Actor)
327 if err != nil {
328 return err
329 }
330 op, err := b.OpenRaw(
331 author,
332 item.ReopenedEvent.CreatedAt.Unix(),
333 map[string]string{metaKeyGithubId: id},
334 )
335
336 if err != nil {
337 return err
338 }
339
340 gi.out <- core.NewImportStatusChange(op.Id())
341 return nil
342
343 case "RenamedTitleEvent":
344 id := parseId(item.RenamedTitleEvent.Id)
345 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
346 if err != cache.ErrNoMatchingOp {
347 return err
348 }
349 if err == nil {
350 return nil
351 }
352 author, err := gi.ensurePerson(repo, item.RenamedTitleEvent.Actor)
353 if err != nil {
354 return err
355 }
356 op, err := b.SetTitleRaw(
357 author,
358 item.RenamedTitleEvent.CreatedAt.Unix(),
359 string(item.RenamedTitleEvent.CurrentTitle),
360 map[string]string{metaKeyGithubId: id},
361 )
362 if err != nil {
363 return err
364 }
365
366 gi.out <- core.NewImportTitleEdition(op.Id())
367 return nil
368 }
369
370 return nil
371}
372
373func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache.BugCache, item issueComment, edits []userContentEdit) error {
374 // ensure person
375 author, err := gi.ensurePerson(repo, item.Author)
376 if err != nil {
377 return err
378 }
379
380 targetOpID, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(item.Id))
381 if err != nil && err != cache.ErrNoMatchingOp {
382 // real error
383 return err
384 }
385
386 // if no edits are given we create the comment
387 if len(edits) == 0 {
388 if err == cache.ErrNoMatchingOp {
389 cleanText, err := text.Cleanup(string(item.Body))
390 if err != nil {
391 return err
392 }
393
394 // add comment operation
395 op, err := b.AddCommentRaw(
396 author,
397 item.CreatedAt.Unix(),
398 cleanText,
399 nil,
400 map[string]string{
401 metaKeyGithubId: parseId(item.Id),
402 metaKeyGithubUrl: parseId(item.Url.String()),
403 },
404 )
405 if err != nil {
406 return err
407 }
408
409 gi.out <- core.NewImportComment(op.Id())
410 return nil
411 }
412
413 } else {
414 for i, edit := range edits {
415 if i == 0 && targetOpID != "" {
416 // The first edit in the github result is the comment creation itself, we already have that
417 continue
418 }
419
420 // ensure editor identity
421 editor, err := gi.ensurePerson(repo, edit.Editor)
422 if err != nil {
423 return err
424 }
425
426 // create comment when target is empty
427 if targetOpID == "" {
428 cleanText, err := text.Cleanup(string(*edit.Diff))
429 if err != nil {
430 return err
431 }
432
433 op, err := b.AddCommentRaw(
434 editor,
435 edit.CreatedAt.Unix(),
436 cleanText,
437 nil,
438 map[string]string{
439 metaKeyGithubId: parseId(item.Id),
440 metaKeyGithubUrl: item.Url.String(),
441 },
442 )
443 if err != nil {
444 return err
445 }
446 gi.out <- core.NewImportComment(op.Id())
447
448 // set target for the nexr edit now that the comment is created
449 targetOpID = op.Id()
450 continue
451 }
452
453 err = gi.ensureCommentEdit(repo, b, targetOpID, edit)
454 if err != nil {
455 return err
456 }
457 }
458 }
459 return nil
460}
461
462func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugCache, target entity.Id, edit userContentEdit) error {
463 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
464 if err == nil {
465 return nil
466 }
467 if err != cache.ErrNoMatchingOp {
468 // real error
469 return err
470 }
471
472 editor, err := gi.ensurePerson(repo, edit.Editor)
473 if err != nil {
474 return err
475 }
476
477 switch {
478 case edit.DeletedAt != nil:
479 // comment deletion, not supported yet
480 return nil
481
482 case edit.DeletedAt == nil:
483
484 cleanText, err := text.Cleanup(string(*edit.Diff))
485 if err != nil {
486 return err
487 }
488
489 // comment edition
490 op, err := b.EditCommentRaw(
491 editor,
492 edit.CreatedAt.Unix(),
493 target,
494 cleanText,
495 map[string]string{
496 metaKeyGithubId: parseId(edit.Id),
497 },
498 )
499
500 if err != nil {
501 return err
502 }
503
504 gi.out <- core.NewImportCommentEdition(op.Id())
505 return nil
506 }
507 return nil
508}
509
510// ensurePerson create a bug.Person from the Github data
511func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
512 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
513 // in it's UI. So we need a special case to get it.
514 if actor == nil {
515 return gi.getGhost(repo)
516 }
517
518 // Look first in the cache
519 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
520 if err == nil {
521 return i, nil
522 }
523 if entity.IsErrMultipleMatch(err) {
524 return nil, err
525 }
526
527 // importing a new identity
528
529 var name string
530 var email string
531
532 switch actor.Typename {
533 case "User":
534 if actor.User.Name != nil {
535 name = string(*(actor.User.Name))
536 }
537 email = string(actor.User.Email)
538 case "Organization":
539 if actor.Organization.Name != nil {
540 name = string(*(actor.Organization.Name))
541 }
542 if actor.Organization.Email != nil {
543 email = string(*(actor.Organization.Email))
544 }
545 case "Bot":
546 }
547
548 i, err = repo.NewIdentityRaw(
549 name,
550 email,
551 string(actor.Login),
552 string(actor.AvatarUrl),
553 map[string]string{
554 metaKeyGithubLogin: string(actor.Login),
555 },
556 )
557
558 if err != nil {
559 return nil, err
560 }
561
562 gi.out <- core.NewImportIdentity(i.Id())
563 return i, nil
564}
565
566func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, error) {
567 // Look first in the cache
568 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, "ghost")
569 if err == nil {
570 return i, nil
571 }
572 if entity.IsErrMultipleMatch(err) {
573 return nil, err
574 }
575
576 var q ghostQuery
577
578 variables := map[string]interface{}{
579 "login": githubv4.String("ghost"),
580 }
581
582 ctx, cancel := context.WithTimeout(gi.iterator.ctx, defaultTimeout)
583 defer cancel()
584
585 err = gi.client.Query(ctx, &q, variables)
586 if err != nil {
587 return nil, err
588 }
589
590 var name string
591 if q.User.Name != nil {
592 name = string(*q.User.Name)
593 }
594
595 return repo.NewIdentityRaw(
596 name,
597 "",
598 string(q.User.Login),
599 string(q.User.AvatarUrl),
600 map[string]string{
601 metaKeyGithubLogin: string(q.User.Login),
602 },
603 )
604}
605
606// parseId convert the unusable githubv4.ID (an interface{}) into a string
607func parseId(id githubv4.ID) string {
608 return fmt.Sprintf("%v", id)
609}