1package github
2
3import (
4 "context"
5 "fmt"
6 "time"
7
8 "github.com/shurcooL/githubv4"
9
10 "github.com/MichaelMure/git-bug/bridge/core"
11 "github.com/MichaelMure/git-bug/bridge/core/auth"
12 "github.com/MichaelMure/git-bug/bug"
13 "github.com/MichaelMure/git-bug/cache"
14 "github.com/MichaelMure/git-bug/entity"
15 "github.com/MichaelMure/git-bug/util/text"
16)
17
18const EmptyTitlePlaceholder = "<empty string>"
19
20// githubImporter implement the Importer interface
21type githubImporter struct {
22 conf core.Configuration
23
24 // default client
25 client *githubv4.Client
26
27 // mediator to access the Github API
28 mediator *importMediator
29
30 // send only channel
31 out chan<- core.ImportResult
32}
33
34func (gi *githubImporter) Init(_ context.Context, repo *cache.RepoCache, conf core.Configuration) error {
35 gi.conf = conf
36 creds, err := auth.List(repo,
37 auth.WithTarget(target),
38 auth.WithKind(auth.KindToken),
39 auth.WithMeta(auth.MetaKeyLogin, conf[confKeyDefaultLogin]),
40 )
41 if err != nil {
42 return err
43 }
44 if len(creds) <= 0 {
45 return ErrMissingIdentityToken
46 }
47 gi.client = buildClient(creds[0].(*auth.Token))
48
49 return nil
50}
51
52// ImportAll iterate over all the configured repository issues and ensure the creation of the
53// missing issues / timeline items / edits / label events ...
54func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
55 gi.mediator = NewImportMediator(ctx, gi.client, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
56 out := make(chan core.ImportResult)
57 gi.out = out
58
59 go func() {
60 defer close(gi.out)
61 var currBug *cache.BugCache
62 var currEvent ImportEvent
63 var nextEvent ImportEvent
64 var err error
65 for {
66 // An IssueEvent contains the issue in its most recent state. If an issue
67 // has at least one issue edit, then the history of the issue edits is
68 // represented by IssueEditEvents. That is, the unedited (original) issue
69 // might be saved only in the IssueEditEvent following the IssueEvent.
70 // Since we replicate the edit history we need to either use the IssueEvent
71 // (if there are no edits) or the IssueEvent together with its first
72 // IssueEditEvent (if there are edits).
73 // Exactly the same is true for comments and comment edits.
74 // As a consequence we need to look at the current event and one look ahead
75 // event.
76 currEvent = nextEvent
77 if currEvent == nil {
78 currEvent = gi.getEventHandleMsgs()
79 }
80 if currEvent == nil {
81 break
82 }
83 nextEvent = gi.getEventHandleMsgs()
84
85 switch event := currEvent.(type) {
86 case RateLimitingEvent:
87 out <- core.NewImportRateLimiting(event.msg)
88 case IssueEvent:
89 // first: commit what is being held in currBug
90 if err = gi.commit(currBug, out); err != nil {
91 out <- core.NewImportError(err, "")
92 return
93 }
94 // second: create new issue
95 switch next := nextEvent.(type) {
96 case IssueEditEvent:
97 // consuming and using next event
98 nextEvent = nil
99 currBug, err = gi.ensureIssue(ctx, repo, &event.issue, &next.userContentEdit)
100 default:
101 currBug, err = gi.ensureIssue(ctx, repo, &event.issue, nil)
102 }
103 if err != nil {
104 err := fmt.Errorf("issue creation: %v", err)
105 out <- core.NewImportError(err, "")
106 return
107 }
108 case IssueEditEvent:
109 err = gi.ensureIssueEdit(ctx, repo, currBug, event.issueId, &event.userContentEdit)
110 if err != nil {
111 err = fmt.Errorf("issue edit: %v", err)
112 out <- core.NewImportError(err, "")
113 return
114 }
115 case TimelineEvent:
116 if next, ok := nextEvent.(CommentEditEvent); ok && event.Typename == "IssueComment" {
117 // consuming and using next event
118 nextEvent = nil
119 err = gi.ensureComment(ctx, repo, currBug, &event.timelineItem.IssueComment, &next.userContentEdit)
120 } else {
121 err = gi.ensureTimelineItem(ctx, repo, currBug, &event.timelineItem)
122 }
123 if err != nil {
124 err = fmt.Errorf("timeline item creation: %v", err)
125 out <- core.NewImportError(err, "")
126 return
127 }
128 case CommentEditEvent:
129 err = gi.ensureCommentEdit(ctx, repo, currBug, event.commentId, &event.userContentEdit)
130 if err != nil {
131 err = fmt.Errorf("comment edit: %v", err)
132 out <- core.NewImportError(err, "")
133 return
134 }
135 default:
136 panic("Unknown event type")
137 }
138 }
139 // commit what is being held in currBug before returning
140 if err = gi.commit(currBug, out); err != nil {
141 out <- core.NewImportError(err, "")
142 }
143 if err = gi.mediator.Error(); err != nil {
144 gi.out <- core.NewImportError(err, "")
145 }
146 }()
147
148 return out, nil
149}
150
151func (gi *githubImporter) getEventHandleMsgs() ImportEvent {
152 for {
153 // read event from import mediator
154 event := gi.mediator.NextImportEvent()
155 // consume (and use) all rate limiting events
156 if e, ok := event.(RateLimitingEvent); ok {
157 gi.out <- core.NewImportRateLimiting(e.msg)
158 continue
159 }
160 return event
161 }
162}
163
164func (gi *githubImporter) commit(b *cache.BugCache, out chan<- core.ImportResult) error {
165 if b == nil {
166 return nil
167 }
168 if !b.NeedCommit() {
169 out <- core.NewImportNothing(b.Id(), "no imported operation")
170 return nil
171 } else if err := b.Commit(); err != nil {
172 // commit bug state
173 return fmt.Errorf("bug commit: %v", err)
174 }
175 return nil
176}
177
178func (gi *githubImporter) ensureIssue(ctx context.Context, repo *cache.RepoCache, issue *issue, issueEdit *userContentEdit) (*cache.BugCache, error) {
179 author, err := gi.ensurePerson(ctx, repo, issue.Author)
180 if err != nil {
181 return nil, err
182 }
183
184 // resolve bug
185 b, err := repo.ResolveBugMatcher(func(excerpt *cache.BugExcerpt) bool {
186 return excerpt.CreateMetadata[core.MetaKeyOrigin] == target &&
187 excerpt.CreateMetadata[metaKeyGithubId] == parseId(issue.Id)
188 })
189 if err == nil {
190 return b, nil
191 }
192 if err != bug.ErrBugNotExist {
193 return nil, err
194 }
195
196 // At Github there exist issues with seemingly empty titles. An example is
197 // https://github.com/NixOS/nixpkgs/issues/72730 .
198 // The title provided by the GraphQL API actually consists of a space followed by a
199 // zero width space (U+200B). This title would cause the NewBugRaw() function to
200 // return an error: empty title.
201 title := string(issue.Title)
202 if title == " \u200b" { // U+200B == zero width space
203 title = EmptyTitlePlaceholder
204 }
205
206 var textInput string
207 if issueEdit != nil {
208 // use the first issue edit: it represents the bug creation itself
209 textInput = string(*issueEdit.Diff)
210 } else {
211 // if there are no issue edits then the issue struct holds the bug creation
212 textInput = string(issue.Body)
213 }
214 cleanText, err := text.Cleanup(textInput)
215 if err != nil {
216 return nil, err
217 }
218
219 // create bug
220 b, _, err = repo.NewBugRaw(
221 author,
222 issue.CreatedAt.Unix(),
223 title, // TODO: this is the *current* title, not the original one
224 cleanText,
225 nil,
226 map[string]string{
227 core.MetaKeyOrigin: target,
228 metaKeyGithubId: parseId(issue.Id),
229 metaKeyGithubUrl: issue.Url.String(),
230 })
231 if err != nil {
232 return nil, err
233 }
234 // importing a new bug
235 gi.out <- core.NewImportBug(b.Id())
236
237 return b, nil
238}
239
240func (gi *githubImporter) ensureIssueEdit(ctx context.Context, repo *cache.RepoCache, bug *cache.BugCache, ghIssueId githubv4.ID, edit *userContentEdit) error {
241 return gi.ensureCommentEdit(ctx, repo, bug, ghIssueId, edit)
242}
243
244func (gi *githubImporter) ensureTimelineItem(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, item *timelineItem) error {
245
246 switch item.Typename {
247 case "IssueComment":
248 err := gi.ensureComment(ctx, repo, b, &item.IssueComment, nil)
249 if err != nil {
250 return fmt.Errorf("timeline comment creation: %v", err)
251 }
252 return nil
253
254 case "LabeledEvent":
255 id := parseId(item.LabeledEvent.Id)
256 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
257 if err == nil {
258 return nil
259 }
260
261 if err != cache.ErrNoMatchingOp {
262 return err
263 }
264 author, err := gi.ensurePerson(ctx, repo, item.LabeledEvent.Actor)
265 if err != nil {
266 return err
267 }
268 op, err := b.ForceChangeLabelsRaw(
269 author,
270 item.LabeledEvent.CreatedAt.Unix(),
271 []string{
272 string(item.LabeledEvent.Label.Name),
273 },
274 nil,
275 map[string]string{metaKeyGithubId: id},
276 )
277 if err != nil {
278 return err
279 }
280
281 gi.out <- core.NewImportLabelChange(op.Id())
282 return nil
283
284 case "UnlabeledEvent":
285 id := parseId(item.UnlabeledEvent.Id)
286 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
287 if err == nil {
288 return nil
289 }
290 if err != cache.ErrNoMatchingOp {
291 return err
292 }
293 author, err := gi.ensurePerson(ctx, repo, item.UnlabeledEvent.Actor)
294 if err != nil {
295 return err
296 }
297
298 op, err := b.ForceChangeLabelsRaw(
299 author,
300 item.UnlabeledEvent.CreatedAt.Unix(),
301 nil,
302 []string{
303 string(item.UnlabeledEvent.Label.Name),
304 },
305 map[string]string{metaKeyGithubId: id},
306 )
307 if err != nil {
308 return err
309 }
310
311 gi.out <- core.NewImportLabelChange(op.Id())
312 return nil
313
314 case "ClosedEvent":
315 id := parseId(item.ClosedEvent.Id)
316 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
317 if err != cache.ErrNoMatchingOp {
318 return err
319 }
320 if err == nil {
321 return nil
322 }
323 author, err := gi.ensurePerson(ctx, repo, item.ClosedEvent.Actor)
324 if err != nil {
325 return err
326 }
327 op, err := b.CloseRaw(
328 author,
329 item.ClosedEvent.CreatedAt.Unix(),
330 map[string]string{metaKeyGithubId: id},
331 )
332
333 if err != nil {
334 return err
335 }
336
337 gi.out <- core.NewImportStatusChange(op.Id())
338 return nil
339
340 case "ReopenedEvent":
341 id := parseId(item.ReopenedEvent.Id)
342 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
343 if err != cache.ErrNoMatchingOp {
344 return err
345 }
346 if err == nil {
347 return nil
348 }
349 author, err := gi.ensurePerson(ctx, repo, item.ReopenedEvent.Actor)
350 if err != nil {
351 return err
352 }
353 op, err := b.OpenRaw(
354 author,
355 item.ReopenedEvent.CreatedAt.Unix(),
356 map[string]string{metaKeyGithubId: id},
357 )
358
359 if err != nil {
360 return err
361 }
362
363 gi.out <- core.NewImportStatusChange(op.Id())
364 return nil
365
366 case "RenamedTitleEvent":
367 id := parseId(item.RenamedTitleEvent.Id)
368 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
369 if err != cache.ErrNoMatchingOp {
370 return err
371 }
372 if err == nil {
373 return nil
374 }
375 author, err := gi.ensurePerson(ctx, repo, item.RenamedTitleEvent.Actor)
376 if err != nil {
377 return err
378 }
379
380 // At Github there exist issues with seemingly empty titles. An example is
381 // https://github.com/NixOS/nixpkgs/issues/72730 .
382 // The title provided by the GraphQL API actually consists of a space followed
383 // by a zero width space (U+200B). This title would cause the NewBugRaw()
384 // function to return an error: empty title.
385 title := string(item.RenamedTitleEvent.CurrentTitle)
386 if title == " \u200b" { // U+200B == zero width space
387 title = EmptyTitlePlaceholder
388 }
389
390 op, err := b.SetTitleRaw(
391 author,
392 item.RenamedTitleEvent.CreatedAt.Unix(),
393 title,
394 map[string]string{metaKeyGithubId: id},
395 )
396 if err != nil {
397 return err
398 }
399
400 gi.out <- core.NewImportTitleEdition(op.Id())
401 return nil
402 }
403
404 return nil
405}
406
407func (gi *githubImporter) ensureCommentEdit(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, ghTargetId githubv4.ID, edit *userContentEdit) error {
408 // find comment
409 target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(ghTargetId))
410 if err != nil {
411 return err
412 }
413 _, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
414 if err == nil {
415 return nil
416 }
417 if err != cache.ErrNoMatchingOp {
418 // real error
419 return err
420 }
421
422 editor, err := gi.ensurePerson(ctx, repo, edit.Editor)
423 if err != nil {
424 return err
425 }
426
427 if edit.DeletedAt != nil {
428 // comment deletion, not supported yet
429 return nil
430 }
431
432 cleanText, err := text.Cleanup(string(*edit.Diff))
433 if err != nil {
434 return err
435 }
436
437 // comment edition
438 op, err := b.EditCommentRaw(
439 editor,
440 edit.CreatedAt.Unix(),
441 target,
442 cleanText,
443 map[string]string{
444 metaKeyGithubId: parseId(edit.Id),
445 },
446 )
447
448 if err != nil {
449 return err
450 }
451
452 gi.out <- core.NewImportCommentEdition(op.Id())
453 return nil
454}
455
456func (gi *githubImporter) ensureComment(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, comment *issueComment, firstEdit *userContentEdit) error {
457 author, err := gi.ensurePerson(ctx, repo, comment.Author)
458 if err != nil {
459 return err
460 }
461
462 _, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(comment.Id))
463 if err == nil {
464 return nil
465 }
466 if err != cache.ErrNoMatchingOp {
467 // real error
468 return err
469 }
470
471 var textInput string
472 if firstEdit != nil {
473 // use the first comment edit: it represents the comment creation itself
474 textInput = string(*firstEdit.Diff)
475 } else {
476 // if there are not comment edits, then the comment struct holds the comment creation
477 textInput = string(comment.Body)
478 }
479 cleanText, err := text.Cleanup(textInput)
480 if err != nil {
481 return err
482 }
483
484 // add comment operation
485 op, err := b.AddCommentRaw(
486 author,
487 comment.CreatedAt.Unix(),
488 cleanText,
489 nil,
490 map[string]string{
491 metaKeyGithubId: parseId(comment.Id),
492 metaKeyGithubUrl: comment.Url.String(),
493 },
494 )
495 if err != nil {
496 return err
497 }
498
499 gi.out <- core.NewImportComment(op.Id())
500 return nil
501}
502
503// ensurePerson create a bug.Person from the Github data
504func (gi *githubImporter) ensurePerson(ctx context.Context, repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
505 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
506 // in it's UI. So we need a special case to get it.
507 if actor == nil {
508 return gi.getGhost(ctx, repo)
509 }
510
511 // Look first in the cache
512 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
513 if err == nil {
514 return i, nil
515 }
516 if entity.IsErrMultipleMatch(err) {
517 return nil, err
518 }
519
520 // importing a new identity
521 var name string
522 var email string
523
524 switch actor.Typename {
525 case "User":
526 if actor.User.Name != nil {
527 name = string(*(actor.User.Name))
528 }
529 email = string(actor.User.Email)
530 case "Organization":
531 if actor.Organization.Name != nil {
532 name = string(*(actor.Organization.Name))
533 }
534 if actor.Organization.Email != nil {
535 email = string(*(actor.Organization.Email))
536 }
537 case "Bot":
538 }
539
540 // Name is not necessarily set, fallback to login as a name is required in the identity
541 if name == "" {
542 name = string(actor.Login)
543 }
544
545 i, err = repo.NewIdentityRaw(
546 name,
547 email,
548 string(actor.Login),
549 string(actor.AvatarUrl),
550 nil,
551 map[string]string{
552 metaKeyGithubLogin: string(actor.Login),
553 },
554 )
555
556 if err != nil {
557 return nil, err
558 }
559
560 gi.out <- core.NewImportIdentity(i.Id())
561 return i, nil
562}
563
564func (gi *githubImporter) getGhost(ctx context.Context, repo *cache.RepoCache) (*cache.IdentityCache, error) {
565 loginName := "ghost"
566 // Look first in the cache
567 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, loginName)
568 if err == nil {
569 return i, nil
570 }
571 if entity.IsErrMultipleMatch(err) {
572 return nil, err
573 }
574 user, err := gi.mediator.User(ctx, loginName)
575 if err != nil {
576 return nil, err
577 }
578 userName := ""
579 if user.Name != nil {
580 userName = string(*user.Name)
581 }
582 return repo.NewIdentityRaw(
583 userName,
584 "",
585 string(user.Login),
586 string(user.AvatarUrl),
587 nil,
588 map[string]string{
589 metaKeyGithubLogin: string(user.Login),
590 },
591 )
592}
593
594// parseId converts the unusable githubv4.ID (an interface{}) into a string
595func parseId(id githubv4.ID) string {
596 return fmt.Sprintf("%v", id)
597}