1package github
2
3import (
4 "context"
5 "fmt"
6 "time"
7
8 "github.com/shurcooL/githubv4"
9
10 "github.com/MichaelMure/git-bug/bridge/core"
11 "github.com/MichaelMure/git-bug/bridge/core/auth"
12 "github.com/MichaelMure/git-bug/cache"
13 "github.com/MichaelMure/git-bug/entities/bug"
14 "github.com/MichaelMure/git-bug/entity"
15 "github.com/MichaelMure/git-bug/util/text"
16)
17
18const EmptyTitlePlaceholder = "<empty string>"
19
20// githubImporter implement the Importer interface
21type githubImporter struct {
22 conf core.Configuration
23
24 // default client
25 client *rateLimitHandlerClient
26
27 // mediator to access the Github API
28 mediator *importMediator
29
30 // send only channel
31 out chan<- core.ImportResult
32}
33
34func (gi *githubImporter) Init(_ context.Context, repo *cache.RepoCache, conf core.Configuration) error {
35 gi.conf = conf
36 creds, err := auth.List(repo,
37 auth.WithTarget(target),
38 auth.WithKind(auth.KindToken),
39 auth.WithMeta(auth.MetaKeyLogin, conf[confKeyDefaultLogin]),
40 )
41 if err != nil {
42 return err
43 }
44 if len(creds) <= 0 {
45 return ErrMissingIdentityToken
46 }
47 gi.client = buildClient(creds[0].(*auth.Token))
48
49 return nil
50}
51
52// ImportAll iterate over all the configured repository issues and ensure the creation of the
53// missing issues / timeline items / edits / label events ...
54func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
55 gi.mediator = NewImportMediator(ctx, gi.client, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
56 out := make(chan core.ImportResult)
57 gi.out = out
58
59 go func() {
60 defer close(gi.out)
61 var currBug *cache.BugCache
62 var currEvent ImportEvent
63 var nextEvent ImportEvent
64 var err error
65 for {
66 // An IssueEvent contains the issue in its most recent state. If an issue
67 // has at least one issue edit, then the history of the issue edits is
68 // represented by IssueEditEvents. That is, the unedited (original) issue
69 // might be saved only in the IssueEditEvent following the IssueEvent.
70 // Since we replicate the edit history we need to either use the IssueEvent
71 // (if there are no edits) or the IssueEvent together with its first
72 // IssueEditEvent (if there are edits).
73 // Exactly the same is true for comments and comment edits.
74 // As a consequence we need to look at the current event and one look ahead
75 // event.
76
77 currEvent = nextEvent
78 if currEvent == nil {
79 currEvent = gi.getEventHandleMsgs()
80 }
81 if currEvent == nil {
82 break
83 }
84 nextEvent = gi.getEventHandleMsgs()
85
86 switch event := currEvent.(type) {
87 case RateLimitingEvent:
88 out <- core.NewImportRateLimiting(event.msg)
89 case IssueEvent:
90 // first: commit what is being held in currBug
91 if err = gi.commit(currBug, out); err != nil {
92 out <- core.NewImportError(err, "")
93 return
94 }
95 // second: create new issue
96 switch next := nextEvent.(type) {
97 case IssueEditEvent:
98 // consuming and using next event
99 nextEvent = nil
100 currBug, err = gi.ensureIssue(ctx, repo, &event.issue, &next.userContentEdit)
101 default:
102 currBug, err = gi.ensureIssue(ctx, repo, &event.issue, nil)
103 }
104 if err != nil {
105 err := fmt.Errorf("issue creation: %v", err)
106 out <- core.NewImportError(err, "")
107 return
108 }
109 case IssueEditEvent:
110 err = gi.ensureIssueEdit(ctx, repo, currBug, event.issueId, &event.userContentEdit)
111 if err != nil {
112 err = fmt.Errorf("issue edit: %v", err)
113 out <- core.NewImportError(err, "")
114 return
115 }
116 case TimelineEvent:
117 if next, ok := nextEvent.(CommentEditEvent); ok && event.Typename == "IssueComment" {
118 // consuming and using next event
119 nextEvent = nil
120 err = gi.ensureComment(ctx, repo, currBug, &event.timelineItem.IssueComment, &next.userContentEdit)
121 } else {
122 err = gi.ensureTimelineItem(ctx, repo, currBug, &event.timelineItem)
123 }
124 if err != nil {
125 err = fmt.Errorf("timeline item creation: %v", err)
126 out <- core.NewImportError(err, "")
127 return
128 }
129 case CommentEditEvent:
130 err = gi.ensureCommentEdit(ctx, repo, currBug, event.commentId, &event.userContentEdit)
131 if err != nil {
132 err = fmt.Errorf("comment edit: %v", err)
133 out <- core.NewImportError(err, "")
134 return
135 }
136 default:
137 panic("Unknown event type")
138 }
139 }
140 // commit what is being held in currBug before returning
141 if err = gi.commit(currBug, out); err != nil {
142 out <- core.NewImportError(err, "")
143 }
144 if err = gi.mediator.Error(); err != nil {
145 gi.out <- core.NewImportError(err, "")
146 }
147 }()
148
149 return out, nil
150}
151
152func (gi *githubImporter) getEventHandleMsgs() ImportEvent {
153 for {
154 // read event from import mediator
155 event := gi.mediator.NextImportEvent()
156 // consume (and use) all rate limiting events
157 if e, ok := event.(RateLimitingEvent); ok {
158 gi.out <- core.NewImportRateLimiting(e.msg)
159 continue
160 }
161 return event
162 }
163}
164
165func (gi *githubImporter) commit(b *cache.BugCache, out chan<- core.ImportResult) error {
166 if b == nil {
167 return nil
168 }
169 if !b.NeedCommit() {
170 out <- core.NewImportNothing(b.Id(), "no imported operation")
171 return nil
172 } else if err := b.Commit(); err != nil {
173 // commit bug state
174 return fmt.Errorf("bug commit: %v", err)
175 }
176 return nil
177}
178
179func (gi *githubImporter) ensureIssue(ctx context.Context, repo *cache.RepoCache, issue *issue, issueEdit *userContentEdit) (*cache.BugCache, error) {
180 author, err := gi.ensurePerson(ctx, repo, issue.Author)
181 if err != nil {
182 return nil, err
183 }
184
185 // resolve bug
186 b, err := repo.ResolveBugMatcher(func(excerpt *cache.BugExcerpt) bool {
187 return excerpt.CreateMetadata[metaKeyGithubUrl] == issue.Url.String() &&
188 excerpt.CreateMetadata[metaKeyGithubId] == parseId(issue.Id)
189 })
190 if err == nil {
191 return b, nil
192 }
193 if err != bug.ErrBugNotExist {
194 return nil, err
195 }
196
197 // At Github there exist issues with seemingly empty titles. An example is
198 // https://github.com/NixOS/nixpkgs/issues/72730 (here the title is actually
199 // a zero width space U+200B).
200 // Set title to some non-empty string, since git-bug does not accept empty titles.
201 title := text.CleanupOneLine(string(issue.Title))
202 if text.Empty(title) {
203 title = EmptyTitlePlaceholder
204 }
205
206 var textInput string
207 if issueEdit != nil {
208 // use the first issue edit: it represents the bug creation itself
209 textInput = string(*issueEdit.Diff)
210 } else {
211 // if there are no issue edits then the issue struct holds the bug creation
212 textInput = string(issue.Body)
213 }
214
215 // create bug
216 b, _, err = repo.NewBugRaw(
217 author,
218 issue.CreatedAt.Unix(),
219 text.CleanupOneLine(title), // TODO: this is the *current* title, not the original one
220 text.Cleanup(textInput),
221 nil,
222 map[string]string{
223 core.MetaKeyOrigin: target,
224 metaKeyGithubId: parseId(issue.Id),
225 metaKeyGithubUrl: issue.Url.String(),
226 })
227 if err != nil {
228 return nil, err
229 }
230 // importing a new bug
231 gi.out <- core.NewImportBug(b.Id())
232
233 return b, nil
234}
235
236func (gi *githubImporter) ensureIssueEdit(ctx context.Context, repo *cache.RepoCache, bug *cache.BugCache, ghIssueId githubv4.ID, edit *userContentEdit) error {
237 return gi.ensureCommentEdit(ctx, repo, bug, ghIssueId, edit)
238}
239
240func (gi *githubImporter) ensureTimelineItem(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, item *timelineItem) error {
241
242 switch item.Typename {
243 case "IssueComment":
244 err := gi.ensureComment(ctx, repo, b, &item.IssueComment, nil)
245 if err != nil {
246 return fmt.Errorf("timeline comment creation: %v", err)
247 }
248 return nil
249
250 case "LabeledEvent":
251 id := parseId(item.LabeledEvent.Id)
252 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
253 if err == nil {
254 return nil
255 }
256
257 if err != cache.ErrNoMatchingOp {
258 return err
259 }
260 author, err := gi.ensurePerson(ctx, repo, item.LabeledEvent.Actor)
261 if err != nil {
262 return err
263 }
264 op, err := b.ForceChangeLabelsRaw(
265 author,
266 item.LabeledEvent.CreatedAt.Unix(),
267 []string{
268 text.CleanupOneLine(string(item.LabeledEvent.Label.Name)),
269 },
270 nil,
271 map[string]string{metaKeyGithubId: id},
272 )
273 if err != nil {
274 return err
275 }
276
277 gi.out <- core.NewImportLabelChange(b.Id(), op.Id())
278 return nil
279
280 case "UnlabeledEvent":
281 id := parseId(item.UnlabeledEvent.Id)
282 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
283 if err == nil {
284 return nil
285 }
286 if err != cache.ErrNoMatchingOp {
287 return err
288 }
289 author, err := gi.ensurePerson(ctx, repo, item.UnlabeledEvent.Actor)
290 if err != nil {
291 return err
292 }
293
294 op, err := b.ForceChangeLabelsRaw(
295 author,
296 item.UnlabeledEvent.CreatedAt.Unix(),
297 nil,
298 []string{
299 text.CleanupOneLine(string(item.UnlabeledEvent.Label.Name)),
300 },
301 map[string]string{metaKeyGithubId: id},
302 )
303 if err != nil {
304 return err
305 }
306
307 gi.out <- core.NewImportLabelChange(b.Id(), op.Id())
308 return nil
309
310 case "ClosedEvent":
311 id := parseId(item.ClosedEvent.Id)
312 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
313 if err != cache.ErrNoMatchingOp {
314 return err
315 }
316 if err == nil {
317 return nil
318 }
319 author, err := gi.ensurePerson(ctx, repo, item.ClosedEvent.Actor)
320 if err != nil {
321 return err
322 }
323 op, err := b.CloseRaw(
324 author,
325 item.ClosedEvent.CreatedAt.Unix(),
326 map[string]string{metaKeyGithubId: id},
327 )
328
329 if err != nil {
330 return err
331 }
332
333 gi.out <- core.NewImportStatusChange(b.Id(), op.Id())
334 return nil
335
336 case "ReopenedEvent":
337 id := parseId(item.ReopenedEvent.Id)
338 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
339 if err != cache.ErrNoMatchingOp {
340 return err
341 }
342 if err == nil {
343 return nil
344 }
345 author, err := gi.ensurePerson(ctx, repo, item.ReopenedEvent.Actor)
346 if err != nil {
347 return err
348 }
349 op, err := b.OpenRaw(
350 author,
351 item.ReopenedEvent.CreatedAt.Unix(),
352 map[string]string{metaKeyGithubId: id},
353 )
354
355 if err != nil {
356 return err
357 }
358
359 gi.out <- core.NewImportStatusChange(b.Id(), op.Id())
360 return nil
361
362 case "RenamedTitleEvent":
363 id := parseId(item.RenamedTitleEvent.Id)
364 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
365 if err != cache.ErrNoMatchingOp {
366 return err
367 }
368 if err == nil {
369 return nil
370 }
371 author, err := gi.ensurePerson(ctx, repo, item.RenamedTitleEvent.Actor)
372 if err != nil {
373 return err
374 }
375
376 // At Github there exist issues with seemingly empty titles. An example is
377 // https://github.com/NixOS/nixpkgs/issues/72730 (here the title is actually
378 // a zero width space U+200B).
379 // Set title to some non-empty string, since git-bug does not accept empty titles.
380 title := text.CleanupOneLine(string(item.RenamedTitleEvent.CurrentTitle))
381 if text.Empty(title) {
382 title = EmptyTitlePlaceholder
383 }
384
385 op, err := b.SetTitleRaw(
386 author,
387 item.RenamedTitleEvent.CreatedAt.Unix(),
388 title,
389 map[string]string{metaKeyGithubId: id},
390 )
391 if err != nil {
392 return err
393 }
394
395 gi.out <- core.NewImportTitleEdition(b.Id(), op.Id())
396 return nil
397 }
398
399 return nil
400}
401
402func (gi *githubImporter) ensureCommentEdit(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, ghTargetId githubv4.ID, edit *userContentEdit) error {
403 // find comment
404 target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(ghTargetId))
405 if err != nil {
406 return err
407 }
408 // check if the comment edition already exist
409 _, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
410 if err == nil {
411 return nil
412 }
413 if err != cache.ErrNoMatchingOp {
414 // real error
415 return err
416 }
417
418 editor, err := gi.ensurePerson(ctx, repo, edit.Editor)
419 if err != nil {
420 return err
421 }
422
423 if edit.DeletedAt != nil {
424 // comment deletion, not supported yet
425 return nil
426 }
427
428 commentId := entity.CombineIds(b.Id(), target)
429
430 // comment edition
431 _, err = b.EditCommentRaw(
432 editor,
433 edit.CreatedAt.Unix(),
434 commentId,
435 text.Cleanup(string(*edit.Diff)),
436 map[string]string{
437 metaKeyGithubId: parseId(edit.Id),
438 },
439 )
440
441 if err != nil {
442 return err
443 }
444
445 gi.out <- core.NewImportCommentEdition(b.Id(), commentId)
446 return nil
447}
448
449func (gi *githubImporter) ensureComment(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, comment *issueComment, firstEdit *userContentEdit) error {
450 author, err := gi.ensurePerson(ctx, repo, comment.Author)
451 if err != nil {
452 return err
453 }
454
455 _, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(comment.Id))
456 if err == nil {
457 return nil
458 }
459 if err != cache.ErrNoMatchingOp {
460 // real error
461 return err
462 }
463
464 var textInput string
465 if firstEdit != nil {
466 // use the first comment edit: it represents the comment creation itself
467 textInput = string(*firstEdit.Diff)
468 } else {
469 // if there are not comment edits, then the comment struct holds the comment creation
470 textInput = string(comment.Body)
471 }
472
473 // add comment operation
474 commentId, _, err := b.AddCommentRaw(
475 author,
476 comment.CreatedAt.Unix(),
477 text.Cleanup(textInput),
478 nil,
479 map[string]string{
480 metaKeyGithubId: parseId(comment.Id),
481 metaKeyGithubUrl: comment.Url.String(),
482 },
483 )
484 if err != nil {
485 return err
486 }
487
488 gi.out <- core.NewImportComment(b.Id(), commentId)
489 return nil
490}
491
492// ensurePerson create a bug.Person from the Github data
493func (gi *githubImporter) ensurePerson(ctx context.Context, repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
494 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
495 // in it's UI. So we need a special case to get it.
496 if actor == nil {
497 return gi.getGhost(ctx, repo)
498 }
499
500 // Look first in the cache
501 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
502 if err == nil {
503 return i, nil
504 }
505 if entity.IsErrMultipleMatch(err) {
506 return nil, err
507 }
508
509 // importing a new identity
510 var name string
511 var email string
512
513 switch actor.Typename {
514 case "User":
515 if actor.User.Name != nil {
516 name = string(*(actor.User.Name))
517 }
518 email = string(actor.User.Email)
519 case "Organization":
520 if actor.Organization.Name != nil {
521 name = string(*(actor.Organization.Name))
522 }
523 if actor.Organization.Email != nil {
524 email = string(*(actor.Organization.Email))
525 }
526 case "Bot":
527 }
528
529 // Name is not necessarily set, fallback to login as a name is required in the identity
530 if name == "" {
531 name = string(actor.Login)
532 }
533
534 i, err = repo.NewIdentityRaw(
535 name,
536 email,
537 string(actor.Login),
538 string(actor.AvatarUrl),
539 nil,
540 map[string]string{
541 metaKeyGithubLogin: string(actor.Login),
542 },
543 )
544
545 if err != nil {
546 return nil, err
547 }
548
549 gi.out <- core.NewImportIdentity(i.Id())
550 return i, nil
551}
552
553func (gi *githubImporter) getGhost(ctx context.Context, repo *cache.RepoCache) (*cache.IdentityCache, error) {
554 loginName := "ghost"
555 // Look first in the cache
556 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, loginName)
557 if err == nil {
558 return i, nil
559 }
560 if entity.IsErrMultipleMatch(err) {
561 return nil, err
562 }
563 user, err := gi.mediator.User(ctx, loginName)
564 if err != nil {
565 return nil, err
566 }
567 userName := ""
568 if user.Name != nil {
569 userName = string(*user.Name)
570 }
571 return repo.NewIdentityRaw(
572 userName,
573 "",
574 string(user.Login),
575 string(user.AvatarUrl),
576 nil,
577 map[string]string{
578 metaKeyGithubLogin: string(user.Login),
579 },
580 )
581}
582
583// parseId converts the unusable githubv4.ID (an interface{}) into a string
584func parseId(id githubv4.ID) string {
585 return fmt.Sprintf("%v", id)
586}