1package github
2
3import (
4 "context"
5 "fmt"
6 "time"
7
8 "github.com/shurcooL/githubv4"
9
10 "github.com/MichaelMure/git-bug/bridge/core"
11 "github.com/MichaelMure/git-bug/bridge/core/auth"
12 "github.com/MichaelMure/git-bug/bug"
13 "github.com/MichaelMure/git-bug/cache"
14 "github.com/MichaelMure/git-bug/entity"
15 "github.com/MichaelMure/git-bug/util/text"
16)
17
18// githubImporter implement the Importer interface
19type githubImporter struct {
20 conf core.Configuration
21
22 // default client
23 client *githubv4.Client
24
25 // iterator
26 iterator *iterator
27
28 // send only channel
29 out chan<- core.ImportResult
30}
31
32func (gi *githubImporter) Init(_ context.Context, repo *cache.RepoCache, conf core.Configuration) error {
33 gi.conf = conf
34
35 creds, err := auth.List(repo,
36 auth.WithTarget(target),
37 auth.WithKind(auth.KindToken),
38 auth.WithMeta(auth.MetaKeyLogin, conf[confKeyDefaultLogin]),
39 )
40 if err != nil {
41 return err
42 }
43
44 if len(creds) == 0 {
45 return ErrMissingIdentityToken
46 }
47
48 gi.client = buildClient(creds[0].(*auth.Token))
49
50 return nil
51}
52
53// ImportAll iterate over all the configured repository issues and ensure the creation of the
54// missing issues / timeline items / edits / label events ...
55func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
56 gi.iterator = NewIterator(ctx, gi.client, 10, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
57 out := make(chan core.ImportResult)
58 gi.out = out
59
60 go func() {
61 defer close(gi.out)
62
63 // Loop over all matching issues
64 for gi.iterator.NextIssue() {
65 issue := gi.iterator.IssueValue()
66 // create issue
67 b, err := gi.ensureIssue(repo, issue)
68 if err != nil {
69 err := fmt.Errorf("issue creation: %v", err)
70 out <- core.NewImportError(err, "")
71 return
72 }
73
74 // loop over timeline items
75 for gi.iterator.NextTimelineItem() {
76 item := gi.iterator.TimelineItemValue()
77 err := gi.ensureTimelineItem(repo, b, item)
78 if err != nil {
79 err = fmt.Errorf("timeline item creation: %v", err)
80 out <- core.NewImportError(err, "")
81 return
82 }
83 }
84
85 if !b.NeedCommit() {
86 out <- core.NewImportNothing(b.Id(), "no imported operation")
87 } else if err := b.Commit(); err != nil {
88 // commit bug state
89 err = fmt.Errorf("bug commit: %v", err)
90 out <- core.NewImportError(err, "")
91 return
92 }
93 }
94
95 if err := gi.iterator.Error(); err != nil {
96 gi.out <- core.NewImportError(err, "")
97 }
98 }()
99
100 return out, nil
101}
102
103func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline) (*cache.BugCache, error) {
104 // ensure issue author
105 author, err := gi.ensurePerson(repo, issue.Author)
106 if err != nil {
107 return nil, err
108 }
109
110 // resolve bug
111 b, err := repo.ResolveBugCreateMetadata(metaKeyGithubUrl, issue.Url.String())
112 if err != nil && err != bug.ErrBugNotExist {
113 return nil, err
114 }
115
116 // get issue edits
117 var issueEdits []userContentEdit
118 for gi.iterator.NextIssueEdit() {
119 issueEdits = append(issueEdits, gi.iterator.IssueEditValue())
120 }
121
122 // if issueEdits is empty
123 if len(issueEdits) == 0 {
124 if err == bug.ErrBugNotExist {
125 cleanText, err := text.Cleanup(string(issue.Body))
126 if err != nil {
127 return nil, err
128 }
129
130 // create bug
131 b, _, err = repo.NewBugRaw(
132 author,
133 issue.CreatedAt.Unix(),
134 issue.Title,
135 cleanText,
136 nil,
137 map[string]string{
138 core.MetaKeyOrigin: target,
139 metaKeyGithubId: parseId(issue.Id),
140 metaKeyGithubUrl: issue.Url.String(),
141 })
142 if err != nil {
143 return nil, err
144 }
145
146 // importing a new bug
147 gi.out <- core.NewImportBug(b.Id())
148 }
149 } else {
150 // create bug from given issueEdits
151 for i, edit := range issueEdits {
152 if i == 0 && b != nil {
153 // The first edit in the github result is the issue creation itself, we already have that
154 continue
155 }
156
157 cleanText, err := text.Cleanup(string(*edit.Diff))
158 if err != nil {
159 return nil, err
160 }
161
162 // if the bug doesn't exist
163 if b == nil {
164 // we create the bug as soon as we have a legit first edition
165 b, _, err = repo.NewBugRaw(
166 author,
167 issue.CreatedAt.Unix(),
168 issue.Title,
169 cleanText,
170 nil,
171 map[string]string{
172 core.MetaKeyOrigin: target,
173 metaKeyGithubId: parseId(issue.Id),
174 metaKeyGithubUrl: issue.Url.String(),
175 },
176 )
177
178 if err != nil {
179 return nil, err
180 }
181 // importing a new bug
182 gi.out <- core.NewImportBug(b.Id())
183 continue
184 }
185
186 // other edits will be added as CommentEdit operations
187 target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(issue.Id))
188 if err == cache.ErrNoMatchingOp {
189 // original comment is missing somehow, issuing a warning
190 gi.out <- core.NewImportWarning(fmt.Errorf("comment ID %s to edit is missing", parseId(issue.Id)), b.Id())
191 continue
192 }
193 if err != nil {
194 return nil, err
195 }
196
197 err = gi.ensureCommentEdit(repo, b, target, edit)
198 if err != nil {
199 return nil, err
200 }
201 }
202 }
203
204 return b, nil
205}
206
207func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, item timelineItem) error {
208
209 switch item.Typename {
210 case "IssueComment":
211 // collect all comment edits
212 var commentEdits []userContentEdit
213 for gi.iterator.NextCommentEdit() {
214 commentEdits = append(commentEdits, gi.iterator.CommentEditValue())
215 }
216
217 // ensureTimelineComment send import events over out chanel
218 err := gi.ensureTimelineComment(repo, b, item.IssueComment, commentEdits)
219 if err != nil {
220 return fmt.Errorf("timeline comment creation: %v", err)
221 }
222 return nil
223
224 case "LabeledEvent":
225 id := parseId(item.LabeledEvent.Id)
226 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
227 if err == nil {
228 return nil
229 }
230
231 if err != cache.ErrNoMatchingOp {
232 return err
233 }
234 author, err := gi.ensurePerson(repo, item.LabeledEvent.Actor)
235 if err != nil {
236 return err
237 }
238 op, err := b.ForceChangeLabelsRaw(
239 author,
240 item.LabeledEvent.CreatedAt.Unix(),
241 []string{
242 string(item.LabeledEvent.Label.Name),
243 },
244 nil,
245 map[string]string{metaKeyGithubId: id},
246 )
247 if err != nil {
248 return err
249 }
250
251 gi.out <- core.NewImportLabelChange(op.Id())
252 return nil
253
254 case "UnlabeledEvent":
255 id := parseId(item.UnlabeledEvent.Id)
256 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
257 if err == nil {
258 return nil
259 }
260 if err != cache.ErrNoMatchingOp {
261 return err
262 }
263 author, err := gi.ensurePerson(repo, item.UnlabeledEvent.Actor)
264 if err != nil {
265 return err
266 }
267
268 op, err := b.ForceChangeLabelsRaw(
269 author,
270 item.UnlabeledEvent.CreatedAt.Unix(),
271 nil,
272 []string{
273 string(item.UnlabeledEvent.Label.Name),
274 },
275 map[string]string{metaKeyGithubId: id},
276 )
277 if err != nil {
278 return err
279 }
280
281 gi.out <- core.NewImportLabelChange(op.Id())
282 return nil
283
284 case "ClosedEvent":
285 id := parseId(item.ClosedEvent.Id)
286 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
287 if err != cache.ErrNoMatchingOp {
288 return err
289 }
290 if err == nil {
291 return nil
292 }
293 author, err := gi.ensurePerson(repo, item.ClosedEvent.Actor)
294 if err != nil {
295 return err
296 }
297 op, err := b.CloseRaw(
298 author,
299 item.ClosedEvent.CreatedAt.Unix(),
300 map[string]string{metaKeyGithubId: id},
301 )
302
303 if err != nil {
304 return err
305 }
306
307 gi.out <- core.NewImportStatusChange(op.Id())
308 return nil
309
310 case "ReopenedEvent":
311 id := parseId(item.ReopenedEvent.Id)
312 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
313 if err != cache.ErrNoMatchingOp {
314 return err
315 }
316 if err == nil {
317 return nil
318 }
319 author, err := gi.ensurePerson(repo, item.ReopenedEvent.Actor)
320 if err != nil {
321 return err
322 }
323 op, err := b.OpenRaw(
324 author,
325 item.ReopenedEvent.CreatedAt.Unix(),
326 map[string]string{metaKeyGithubId: id},
327 )
328
329 if err != nil {
330 return err
331 }
332
333 gi.out <- core.NewImportStatusChange(op.Id())
334 return nil
335
336 case "RenamedTitleEvent":
337 id := parseId(item.RenamedTitleEvent.Id)
338 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
339 if err != cache.ErrNoMatchingOp {
340 return err
341 }
342 if err == nil {
343 return nil
344 }
345 author, err := gi.ensurePerson(repo, item.RenamedTitleEvent.Actor)
346 if err != nil {
347 return err
348 }
349 op, err := b.SetTitleRaw(
350 author,
351 item.RenamedTitleEvent.CreatedAt.Unix(),
352 string(item.RenamedTitleEvent.CurrentTitle),
353 map[string]string{metaKeyGithubId: id},
354 )
355 if err != nil {
356 return err
357 }
358
359 gi.out <- core.NewImportTitleEdition(op.Id())
360 return nil
361 }
362
363 return nil
364}
365
366func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache.BugCache, item issueComment, edits []userContentEdit) error {
367 // ensure person
368 author, err := gi.ensurePerson(repo, item.Author)
369 if err != nil {
370 return err
371 }
372
373 targetOpID, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(item.Id))
374 if err != nil && err != cache.ErrNoMatchingOp {
375 // real error
376 return err
377 }
378
379 // if no edits are given we create the comment
380 if len(edits) == 0 {
381 if err == cache.ErrNoMatchingOp {
382 cleanText, err := text.Cleanup(string(item.Body))
383 if err != nil {
384 return err
385 }
386
387 // add comment operation
388 op, err := b.AddCommentRaw(
389 author,
390 item.CreatedAt.Unix(),
391 cleanText,
392 nil,
393 map[string]string{
394 metaKeyGithubId: parseId(item.Id),
395 metaKeyGithubUrl: parseId(item.Url.String()),
396 },
397 )
398 if err != nil {
399 return err
400 }
401
402 gi.out <- core.NewImportComment(op.Id())
403 return nil
404 }
405
406 } else {
407 for i, edit := range edits {
408 if i == 0 && targetOpID != "" {
409 // The first edit in the github result is the comment creation itself, we already have that
410 continue
411 }
412
413 // ensure editor identity
414 editor, err := gi.ensurePerson(repo, edit.Editor)
415 if err != nil {
416 return err
417 }
418
419 // create comment when target is empty
420 if targetOpID == "" {
421 cleanText, err := text.Cleanup(string(*edit.Diff))
422 if err != nil {
423 return err
424 }
425
426 op, err := b.AddCommentRaw(
427 editor,
428 edit.CreatedAt.Unix(),
429 cleanText,
430 nil,
431 map[string]string{
432 metaKeyGithubId: parseId(item.Id),
433 metaKeyGithubUrl: item.Url.String(),
434 },
435 )
436 if err != nil {
437 return err
438 }
439 gi.out <- core.NewImportComment(op.Id())
440
441 // set target for the next edit now that the comment is created
442 targetOpID = op.Id()
443 continue
444 }
445
446 err = gi.ensureCommentEdit(repo, b, targetOpID, edit)
447 if err != nil {
448 return err
449 }
450 }
451 }
452 return nil
453}
454
455func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugCache, target entity.Id, edit userContentEdit) error {
456 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
457 if err == nil {
458 return nil
459 }
460 if err != cache.ErrNoMatchingOp {
461 // real error
462 return err
463 }
464
465 editor, err := gi.ensurePerson(repo, edit.Editor)
466 if err != nil {
467 return err
468 }
469
470 switch {
471 case edit.DeletedAt != nil:
472 // comment deletion, not supported yet
473 return nil
474
475 case edit.DeletedAt == nil:
476
477 cleanText, err := text.Cleanup(string(*edit.Diff))
478 if err != nil {
479 return err
480 }
481
482 // comment edition
483 op, err := b.EditCommentRaw(
484 editor,
485 edit.CreatedAt.Unix(),
486 target,
487 cleanText,
488 map[string]string{
489 metaKeyGithubId: parseId(edit.Id),
490 },
491 )
492
493 if err != nil {
494 return err
495 }
496
497 gi.out <- core.NewImportCommentEdition(op.Id())
498 return nil
499 }
500 return nil
501}
502
503// ensurePerson create a bug.Person from the Github data
504func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
505 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
506 // in it's UI. So we need a special case to get it.
507 if actor == nil {
508 return gi.getGhost(repo)
509 }
510
511 // Look first in the cache
512 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
513 if err == nil {
514 return i, nil
515 }
516 if entity.IsErrMultipleMatch(err) {
517 return nil, err
518 }
519
520 // importing a new identity
521
522 var name string
523 var email string
524
525 switch actor.Typename {
526 case "User":
527 if actor.User.Name != nil {
528 name = string(*(actor.User.Name))
529 }
530 email = string(actor.User.Email)
531 case "Organization":
532 if actor.Organization.Name != nil {
533 name = string(*(actor.Organization.Name))
534 }
535 if actor.Organization.Email != nil {
536 email = string(*(actor.Organization.Email))
537 }
538 case "Bot":
539 }
540
541 // Name is not necessarily set, fallback to login as a name is required in the identity
542 if name == "" {
543 name = string(actor.Login)
544 }
545
546 i, err = repo.NewIdentityRaw(
547 name,
548 email,
549 string(actor.AvatarUrl),
550 map[string]string{
551 metaKeyGithubLogin: string(actor.Login),
552 },
553 )
554
555 if err != nil {
556 return nil, err
557 }
558
559 gi.out <- core.NewImportIdentity(i.Id())
560 return i, nil
561}
562
563func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, error) {
564 // Look first in the cache
565 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, "ghost")
566 if err == nil {
567 return i, nil
568 }
569 if entity.IsErrMultipleMatch(err) {
570 return nil, err
571 }
572
573 var q ghostQuery
574
575 variables := map[string]interface{}{
576 "login": githubv4.String("ghost"),
577 }
578
579 ctx, cancel := context.WithTimeout(gi.iterator.ctx, defaultTimeout)
580 defer cancel()
581
582 err = gi.client.Query(ctx, &q, variables)
583 if err != nil {
584 return nil, err
585 }
586
587 var name string
588 if q.User.Name != nil {
589 name = string(*q.User.Name)
590 }
591
592 return repo.NewIdentityRaw(
593 name,
594 "",
595 string(q.User.AvatarUrl),
596 map[string]string{
597 metaKeyGithubLogin: string(q.User.Login),
598 },
599 )
600}
601
602// parseId convert the unusable githubv4.ID (an interface{}) into a string
603func parseId(id githubv4.ID) string {
604 return fmt.Sprintf("%v", id)
605}