1package github
2
3import (
4 "context"
5 "fmt"
6 "time"
7
8 "github.com/shurcooL/githubv4"
9
10 "github.com/MichaelMure/git-bug/bridge/core"
11 "github.com/MichaelMure/git-bug/bridge/core/auth"
12 "github.com/MichaelMure/git-bug/cache"
13 "github.com/MichaelMure/git-bug/entity"
14 "github.com/MichaelMure/git-bug/util/text"
15)
16
17const EmptyTitlePlaceholder = "<empty string>"
18
19// githubImporter implement the Importer interface
20type githubImporter struct {
21 conf core.Configuration
22
23 // default client
24 client *rateLimitHandlerClient
25
26 // mediator to access the Github API
27 mediator *importMediator
28
29 // send only channel
30 out chan<- core.ImportResult
31}
32
33func (gi *githubImporter) Init(_ context.Context, repo *cache.RepoCache, conf core.Configuration) error {
34 gi.conf = conf
35 creds, err := auth.List(repo,
36 auth.WithTarget(target),
37 auth.WithKind(auth.KindToken),
38 auth.WithMeta(auth.MetaKeyLogin, conf[confKeyDefaultLogin]),
39 )
40 if err != nil {
41 return err
42 }
43 if len(creds) <= 0 {
44 return ErrMissingIdentityToken
45 }
46 gi.client = buildClient(creds[0].(*auth.Token))
47
48 return nil
49}
50
51// ImportAll iterate over all the configured repository issues and ensure the creation of the
52// missing issues / timeline items / edits / label events ...
53func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
54 gi.mediator = NewImportMediator(ctx, gi.client, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
55 out := make(chan core.ImportResult)
56 gi.out = out
57
58 go func() {
59 defer close(gi.out)
60 var currBug *cache.BugCache
61 var currEvent ImportEvent
62 var nextEvent ImportEvent
63 var err error
64 for {
65 // An IssueEvent contains the issue in its most recent state. If an issue
66 // has at least one issue edit, then the history of the issue edits is
67 // represented by IssueEditEvents. That is, the unedited (original) issue
68 // might be saved only in the IssueEditEvent following the IssueEvent.
69 // Since we replicate the edit history we need to either use the IssueEvent
70 // (if there are no edits) or the IssueEvent together with its first
71 // IssueEditEvent (if there are edits).
72 // Exactly the same is true for comments and comment edits.
73 // As a consequence we need to look at the current event and one look ahead
74 // event.
75
76 currEvent = nextEvent
77 if currEvent == nil {
78 currEvent = gi.getEventHandleMsgs()
79 }
80 if currEvent == nil {
81 break
82 }
83 nextEvent = gi.getEventHandleMsgs()
84
85 switch event := currEvent.(type) {
86 case RateLimitingEvent:
87 out <- core.NewImportRateLimiting(event.msg)
88 case IssueEvent:
89 // first: commit what is being held in currBug
90 if err = gi.commit(currBug, out); err != nil {
91 out <- core.NewImportError(err, "")
92 return
93 }
94 // second: create new issue
95 switch next := nextEvent.(type) {
96 case IssueEditEvent:
97 // consuming and using next event
98 nextEvent = nil
99 currBug, err = gi.ensureIssue(ctx, repo, &event.issue, &next.userContentEdit)
100 default:
101 currBug, err = gi.ensureIssue(ctx, repo, &event.issue, nil)
102 }
103 if err != nil {
104 err := fmt.Errorf("issue creation: %v", err)
105 out <- core.NewImportError(err, "")
106 return
107 }
108 case IssueEditEvent:
109 err = gi.ensureIssueEdit(ctx, repo, currBug, event.issueId, &event.userContentEdit)
110 if err != nil {
111 err = fmt.Errorf("issue edit: %v", err)
112 out <- core.NewImportError(err, "")
113 return
114 }
115 case TimelineEvent:
116 if next, ok := nextEvent.(CommentEditEvent); ok && event.Typename == "IssueComment" {
117 // consuming and using next event
118 nextEvent = nil
119 err = gi.ensureComment(ctx, repo, currBug, &event.timelineItem.IssueComment, &next.userContentEdit)
120 } else {
121 err = gi.ensureTimelineItem(ctx, repo, currBug, &event.timelineItem)
122 }
123 if err != nil {
124 err = fmt.Errorf("timeline item creation: %v", err)
125 out <- core.NewImportError(err, "")
126 return
127 }
128 case CommentEditEvent:
129 err = gi.ensureCommentEdit(ctx, repo, currBug, event.commentId, &event.userContentEdit)
130 if err != nil {
131 err = fmt.Errorf("comment edit: %v", err)
132 out <- core.NewImportError(err, "")
133 return
134 }
135 default:
136 panic("Unknown event type")
137 }
138 }
139 // commit what is being held in currBug before returning
140 if err = gi.commit(currBug, out); err != nil {
141 out <- core.NewImportError(err, "")
142 }
143 if err = gi.mediator.Error(); err != nil {
144 gi.out <- core.NewImportError(err, "")
145 }
146 }()
147
148 return out, nil
149}
150
151func (gi *githubImporter) getEventHandleMsgs() ImportEvent {
152 for {
153 // read event from import mediator
154 event := gi.mediator.NextImportEvent()
155 // consume (and use) all rate limiting events
156 if e, ok := event.(RateLimitingEvent); ok {
157 gi.out <- core.NewImportRateLimiting(e.msg)
158 continue
159 }
160 return event
161 }
162}
163
164func (gi *githubImporter) commit(b *cache.BugCache, out chan<- core.ImportResult) error {
165 if b == nil {
166 return nil
167 }
168 if !b.NeedCommit() {
169 out <- core.NewImportNothing(b.Id(), "no imported operation")
170 return nil
171 } else if err := b.Commit(); err != nil {
172 // commit bug state
173 return fmt.Errorf("bug commit: %v", err)
174 }
175 return nil
176}
177
178func (gi *githubImporter) ensureIssue(ctx context.Context, repo *cache.RepoCache, issue *issue, issueEdit *userContentEdit) (*cache.BugCache, error) {
179 author, err := gi.ensurePerson(ctx, repo, issue.Author)
180 if err != nil {
181 return nil, err
182 }
183
184 // resolve bug
185 b, err := repo.Bugs().ResolveMatcher(func(excerpt *cache.BugExcerpt) bool {
186 return excerpt.CreateMetadata[metaKeyGithubUrl] == issue.Url.String() &&
187 excerpt.CreateMetadata[metaKeyGithubId] == parseId(issue.Id)
188 })
189 if err == nil {
190 return b, nil
191 }
192 if !entity.IsErrNotFound(err) {
193 return nil, err
194 }
195
196 // At Github there exist issues with seemingly empty titles. An example is
197 // https://github.com/NixOS/nixpkgs/issues/72730 (here the title is actually
198 // a zero width space U+200B).
199 // Set title to some non-empty string, since git-bug does not accept empty titles.
200 title := text.CleanupOneLine(string(issue.Title))
201 if text.Empty(title) {
202 title = EmptyTitlePlaceholder
203 }
204
205 var textInput string
206 if issueEdit != nil {
207 // use the first issue edit: it represents the bug creation itself
208 textInput = string(*issueEdit.Diff)
209 } else {
210 // if there are no issue edits then the issue struct holds the bug creation
211 textInput = string(issue.Body)
212 }
213
214 // create bug
215 b, _, err = repo.Bugs().NewRaw(
216 author,
217 issue.CreatedAt.Unix(),
218 text.CleanupOneLine(title), // TODO: this is the *current* title, not the original one
219 text.Cleanup(textInput),
220 nil,
221 map[string]string{
222 core.MetaKeyOrigin: target,
223 metaKeyGithubId: parseId(issue.Id),
224 metaKeyGithubUrl: issue.Url.String(),
225 })
226 if err != nil {
227 return nil, err
228 }
229 // importing a new bug
230 gi.out <- core.NewImportBug(b.Id())
231
232 return b, nil
233}
234
235func (gi *githubImporter) ensureIssueEdit(ctx context.Context, repo *cache.RepoCache, bug *cache.BugCache, ghIssueId githubv4.ID, edit *userContentEdit) error {
236 return gi.ensureCommentEdit(ctx, repo, bug, ghIssueId, edit)
237}
238
239func (gi *githubImporter) ensureTimelineItem(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, item *timelineItem) error {
240
241 switch item.Typename {
242 case "IssueComment":
243 err := gi.ensureComment(ctx, repo, b, &item.IssueComment, nil)
244 if err != nil {
245 return fmt.Errorf("timeline comment creation: %v", err)
246 }
247 return nil
248
249 case "LabeledEvent":
250 id := parseId(item.LabeledEvent.Id)
251 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
252 if err == nil {
253 return nil
254 }
255
256 if err != cache.ErrNoMatchingOp {
257 return err
258 }
259 author, err := gi.ensurePerson(ctx, repo, item.LabeledEvent.Actor)
260 if err != nil {
261 return err
262 }
263 op, err := b.ForceChangeLabelsRaw(
264 author,
265 item.LabeledEvent.CreatedAt.Unix(),
266 []string{
267 text.CleanupOneLine(string(item.LabeledEvent.Label.Name)),
268 },
269 nil,
270 map[string]string{metaKeyGithubId: id},
271 )
272 if err != nil {
273 return err
274 }
275
276 gi.out <- core.NewImportLabelChange(b.Id(), op.Id())
277 return nil
278
279 case "UnlabeledEvent":
280 id := parseId(item.UnlabeledEvent.Id)
281 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
282 if err == nil {
283 return nil
284 }
285 if err != cache.ErrNoMatchingOp {
286 return err
287 }
288 author, err := gi.ensurePerson(ctx, repo, item.UnlabeledEvent.Actor)
289 if err != nil {
290 return err
291 }
292
293 op, err := b.ForceChangeLabelsRaw(
294 author,
295 item.UnlabeledEvent.CreatedAt.Unix(),
296 nil,
297 []string{
298 text.CleanupOneLine(string(item.UnlabeledEvent.Label.Name)),
299 },
300 map[string]string{metaKeyGithubId: id},
301 )
302 if err != nil {
303 return err
304 }
305
306 gi.out <- core.NewImportLabelChange(b.Id(), op.Id())
307 return nil
308
309 case "ClosedEvent":
310 id := parseId(item.ClosedEvent.Id)
311 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
312 if err != cache.ErrNoMatchingOp {
313 return err
314 }
315 if err == nil {
316 return nil
317 }
318 author, err := gi.ensurePerson(ctx, repo, item.ClosedEvent.Actor)
319 if err != nil {
320 return err
321 }
322 op, err := b.CloseRaw(
323 author,
324 item.ClosedEvent.CreatedAt.Unix(),
325 map[string]string{metaKeyGithubId: id},
326 )
327
328 if err != nil {
329 return err
330 }
331
332 gi.out <- core.NewImportStatusChange(b.Id(), op.Id())
333 return nil
334
335 case "ReopenedEvent":
336 id := parseId(item.ReopenedEvent.Id)
337 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
338 if err != cache.ErrNoMatchingOp {
339 return err
340 }
341 if err == nil {
342 return nil
343 }
344 author, err := gi.ensurePerson(ctx, repo, item.ReopenedEvent.Actor)
345 if err != nil {
346 return err
347 }
348 op, err := b.OpenRaw(
349 author,
350 item.ReopenedEvent.CreatedAt.Unix(),
351 map[string]string{metaKeyGithubId: id},
352 )
353
354 if err != nil {
355 return err
356 }
357
358 gi.out <- core.NewImportStatusChange(b.Id(), op.Id())
359 return nil
360
361 case "RenamedTitleEvent":
362 id := parseId(item.RenamedTitleEvent.Id)
363 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
364 if err != cache.ErrNoMatchingOp {
365 return err
366 }
367 if err == nil {
368 return nil
369 }
370 author, err := gi.ensurePerson(ctx, repo, item.RenamedTitleEvent.Actor)
371 if err != nil {
372 return err
373 }
374
375 // At Github there exist issues with seemingly empty titles. An example is
376 // https://github.com/NixOS/nixpkgs/issues/72730 (here the title is actually
377 // a zero width space U+200B).
378 // Set title to some non-empty string, since git-bug does not accept empty titles.
379 title := text.CleanupOneLine(string(item.RenamedTitleEvent.CurrentTitle))
380 if text.Empty(title) {
381 title = EmptyTitlePlaceholder
382 }
383
384 op, err := b.SetTitleRaw(
385 author,
386 item.RenamedTitleEvent.CreatedAt.Unix(),
387 title,
388 map[string]string{metaKeyGithubId: id},
389 )
390 if err != nil {
391 return err
392 }
393
394 gi.out <- core.NewImportTitleEdition(b.Id(), op.Id())
395 return nil
396 }
397
398 return nil
399}
400
401func (gi *githubImporter) ensureCommentEdit(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, ghTargetId githubv4.ID, edit *userContentEdit) error {
402 // find comment
403 target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(ghTargetId))
404 if err != nil {
405 return err
406 }
407 // check if the comment edition already exist
408 _, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
409 if err == nil {
410 return nil
411 }
412 if err != cache.ErrNoMatchingOp {
413 // real error
414 return err
415 }
416
417 editor, err := gi.ensurePerson(ctx, repo, edit.Editor)
418 if err != nil {
419 return err
420 }
421
422 if edit.DeletedAt != nil {
423 // comment deletion, not supported yet
424 return nil
425 }
426
427 commentId := entity.CombineIds(b.Id(), target)
428
429 // comment edition
430 _, err = b.EditCommentRaw(
431 editor,
432 edit.CreatedAt.Unix(),
433 commentId,
434 text.Cleanup(string(*edit.Diff)),
435 map[string]string{
436 metaKeyGithubId: parseId(edit.Id),
437 },
438 )
439
440 if err != nil {
441 return err
442 }
443
444 gi.out <- core.NewImportCommentEdition(b.Id(), commentId)
445 return nil
446}
447
448func (gi *githubImporter) ensureComment(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, comment *issueComment, firstEdit *userContentEdit) error {
449 author, err := gi.ensurePerson(ctx, repo, comment.Author)
450 if err != nil {
451 return err
452 }
453
454 _, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(comment.Id))
455 if err == nil {
456 return nil
457 }
458 if err != cache.ErrNoMatchingOp {
459 // real error
460 return err
461 }
462
463 var textInput string
464 if firstEdit != nil {
465 // use the first comment edit: it represents the comment creation itself
466 textInput = string(*firstEdit.Diff)
467 } else {
468 // if there are not comment edits, then the comment struct holds the comment creation
469 textInput = string(comment.Body)
470 }
471
472 // add comment operation
473 commentId, _, err := b.AddCommentRaw(
474 author,
475 comment.CreatedAt.Unix(),
476 text.Cleanup(textInput),
477 nil,
478 map[string]string{
479 metaKeyGithubId: parseId(comment.Id),
480 metaKeyGithubUrl: comment.Url.String(),
481 },
482 )
483 if err != nil {
484 return err
485 }
486
487 gi.out <- core.NewImportComment(b.Id(), commentId)
488 return nil
489}
490
491// ensurePerson create a bug.Person from the Github data
492func (gi *githubImporter) ensurePerson(ctx context.Context, repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
493 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
494 // in it's UI. So we need a special case to get it.
495 if actor == nil {
496 return gi.getGhost(ctx, repo)
497 }
498
499 // Look first in the cache
500 i, err := repo.Identities().ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
501 if err == nil {
502 return i, nil
503 }
504 if entity.IsErrMultipleMatch(err) {
505 return nil, err
506 }
507
508 // importing a new identity
509 var name string
510 var email string
511
512 switch actor.Typename {
513 case "User":
514 if actor.User.Name != nil {
515 name = string(*(actor.User.Name))
516 }
517 email = string(actor.User.Email)
518 case "Organization":
519 if actor.Organization.Name != nil {
520 name = string(*(actor.Organization.Name))
521 }
522 if actor.Organization.Email != nil {
523 email = string(*(actor.Organization.Email))
524 }
525 case "Bot":
526 }
527
528 // Name is not necessarily set, fallback to login as a name is required in the identity
529 if name == "" {
530 name = string(actor.Login)
531 }
532
533 i, err = repo.Identities().NewRaw(
534 name,
535 email,
536 string(actor.Login),
537 string(actor.AvatarUrl),
538 nil,
539 map[string]string{
540 metaKeyGithubLogin: string(actor.Login),
541 },
542 )
543
544 if err != nil {
545 return nil, err
546 }
547
548 gi.out <- core.NewImportIdentity(i.Id())
549 return i, nil
550}
551
552func (gi *githubImporter) getGhost(ctx context.Context, repo *cache.RepoCache) (*cache.IdentityCache, error) {
553 loginName := "ghost"
554 // Look first in the cache
555 i, err := repo.Identities().ResolveIdentityImmutableMetadata(metaKeyGithubLogin, loginName)
556 if err == nil {
557 return i, nil
558 }
559 if entity.IsErrMultipleMatch(err) {
560 return nil, err
561 }
562 user, err := gi.mediator.User(ctx, loginName)
563 if err != nil {
564 return nil, err
565 }
566 userName := ""
567 if user.Name != nil {
568 userName = string(*user.Name)
569 }
570 return repo.Identities().NewRaw(
571 userName,
572 "",
573 string(user.Login),
574 string(user.AvatarUrl),
575 nil,
576 map[string]string{
577 metaKeyGithubLogin: string(user.Login),
578 },
579 )
580}
581
582// parseId converts the unusable githubv4.ID (an interface{}) into a string
583func parseId(id githubv4.ID) string {
584 return fmt.Sprintf("%v", id)
585}