1package github
2
3import (
4 "context"
5 "fmt"
6 "time"
7
8 "github.com/shurcooL/githubv4"
9
10 "github.com/MichaelMure/git-bug/bridge/core"
11 "github.com/MichaelMure/git-bug/bridge/core/auth"
12 "github.com/MichaelMure/git-bug/cache"
13 "github.com/MichaelMure/git-bug/entities/bug"
14 "github.com/MichaelMure/git-bug/entity"
15 "github.com/MichaelMure/git-bug/util/text"
16)
17
18const EmptyTitlePlaceholder = "<empty string>"
19
20// githubImporter implement the Importer interface
21type githubImporter struct {
22 conf core.Configuration
23
24 // default client
25 client *rateLimitHandlerClient
26
27 // mediator to access the Github API
28 mediator *importMediator
29
30 // send only channel
31 out chan<- core.ImportResult
32}
33
34func (gi *githubImporter) Init(_ context.Context, repo *cache.RepoCache, conf core.Configuration) error {
35 gi.conf = conf
36 creds, err := auth.List(repo,
37 auth.WithTarget(target),
38 auth.WithKind(auth.KindToken),
39 auth.WithMeta(auth.MetaKeyLogin, conf[confKeyDefaultLogin]),
40 )
41 if err != nil {
42 return err
43 }
44 if len(creds) <= 0 {
45 return ErrMissingIdentityToken
46 }
47 gi.client = buildClient(creds[0].(*auth.Token))
48
49 return nil
50}
51
52// ImportAll iterate over all the configured repository issues and ensure the creation of the
53// missing issues / timeline items / edits / label events ...
54func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
55 gi.mediator = NewImportMediator(ctx, gi.client, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
56 out := make(chan core.ImportResult)
57 gi.out = out
58
59 go func() {
60 defer close(gi.out)
61 var currBug *cache.BugCache
62 var currEvent ImportEvent
63 var nextEvent ImportEvent
64 var err error
65 for {
66 // An IssueEvent contains the issue in its most recent state. If an issue
67 // has at least one issue edit, then the history of the issue edits is
68 // represented by IssueEditEvents. That is, the unedited (original) issue
69 // might be saved only in the IssueEditEvent following the IssueEvent.
70 // Since we replicate the edit history we need to either use the IssueEvent
71 // (if there are no edits) or the IssueEvent together with its first
72 // IssueEditEvent (if there are edits).
73 // Exactly the same is true for comments and comment edits.
74 // As a consequence we need to look at the current event and one look ahead
75 // event.
76
77 currEvent = nextEvent
78 if currEvent == nil {
79 currEvent = gi.getEventHandleMsgs()
80 }
81 if currEvent == nil {
82 break
83 }
84 nextEvent = gi.getEventHandleMsgs()
85
86 switch event := currEvent.(type) {
87 case RateLimitingEvent:
88 out <- core.NewImportRateLimiting(event.msg)
89 case IssueEvent:
90 // first: commit what is being held in currBug
91 if err = gi.commit(currBug, out); err != nil {
92 out <- core.NewImportError(err, "")
93 return
94 }
95 // second: create new issue
96 switch next := nextEvent.(type) {
97 case IssueEditEvent:
98 // consuming and using next event
99 nextEvent = nil
100 currBug, err = gi.ensureIssue(ctx, repo, &event.issue, &next.userContentEdit)
101 default:
102 currBug, err = gi.ensureIssue(ctx, repo, &event.issue, nil)
103 }
104 if err != nil {
105 err := fmt.Errorf("issue creation: %v", err)
106 out <- core.NewImportError(err, "")
107 return
108 }
109 case IssueEditEvent:
110 err = gi.ensureIssueEdit(ctx, repo, currBug, event.issueId, &event.userContentEdit)
111 if err != nil {
112 err = fmt.Errorf("issue edit: %v", err)
113 out <- core.NewImportError(err, "")
114 return
115 }
116 case TimelineEvent:
117 if next, ok := nextEvent.(CommentEditEvent); ok && event.Typename == "IssueComment" {
118 // consuming and using next event
119 nextEvent = nil
120 err = gi.ensureComment(ctx, repo, currBug, &event.timelineItem.IssueComment, &next.userContentEdit)
121 } else {
122 err = gi.ensureTimelineItem(ctx, repo, currBug, &event.timelineItem)
123 }
124 if err != nil {
125 err = fmt.Errorf("timeline item creation: %v", err)
126 out <- core.NewImportError(err, "")
127 return
128 }
129 case CommentEditEvent:
130 err = gi.ensureCommentEdit(ctx, repo, currBug, event.commentId, &event.userContentEdit)
131 if err != nil {
132 err = fmt.Errorf("comment edit: %v", err)
133 out <- core.NewImportError(err, "")
134 return
135 }
136 default:
137 panic("Unknown event type")
138 }
139 }
140 // commit what is being held in currBug before returning
141 if err = gi.commit(currBug, out); err != nil {
142 out <- core.NewImportError(err, "")
143 }
144 if err = gi.mediator.Error(); err != nil {
145 gi.out <- core.NewImportError(err, "")
146 }
147 }()
148
149 return out, nil
150}
151
152func (gi *githubImporter) getEventHandleMsgs() ImportEvent {
153 for {
154 // read event from import mediator
155 event := gi.mediator.NextImportEvent()
156 // consume (and use) all rate limiting events
157 if e, ok := event.(RateLimitingEvent); ok {
158 gi.out <- core.NewImportRateLimiting(e.msg)
159 continue
160 }
161 return event
162 }
163}
164
165func (gi *githubImporter) commit(b *cache.BugCache, out chan<- core.ImportResult) error {
166 if b == nil {
167 return nil
168 }
169 if !b.NeedCommit() {
170 out <- core.NewImportNothing(b.Id(), "no imported operation")
171 return nil
172 } else if err := b.Commit(); err != nil {
173 // commit bug state
174 return fmt.Errorf("bug commit: %v", err)
175 }
176 return nil
177}
178
179func (gi *githubImporter) ensureIssue(ctx context.Context, repo *cache.RepoCache, issue *issue, issueEdit *userContentEdit) (*cache.BugCache, error) {
180 author, err := gi.ensurePerson(ctx, repo, issue.Author)
181 if err != nil {
182 return nil, err
183 }
184
185 // resolve bug
186 b, err := repo.ResolveBugMatcher(func(excerpt *cache.BugExcerpt) bool {
187 return excerpt.CreateMetadata[metaKeyGithubUrl] == issue.Url.String() &&
188 excerpt.CreateMetadata[metaKeyGithubId] == parseId(issue.Id)
189 })
190 if err == nil {
191 return b, nil
192 }
193 if err != bug.ErrBugNotExist {
194 return nil, err
195 }
196
197 // At Github there exist issues with seemingly empty titles. An example is
198 // https://github.com/NixOS/nixpkgs/issues/72730 (here the title is actually
199 // a zero width space U+200B).
200 // Set title to some non-empty string, since git-bug does not accept empty titles.
201 title := text.CleanupOneLine(string(issue.Title))
202 if text.Empty(title) {
203 title = EmptyTitlePlaceholder
204 }
205
206 var textInput string
207 if issueEdit != nil {
208 // use the first issue edit: it represents the bug creation itself
209 textInput = string(*issueEdit.Diff)
210 } else {
211 // if there are no issue edits then the issue struct holds the bug creation
212 textInput = string(issue.Body)
213 }
214
215 // create bug
216 b, _, err = repo.NewBugRaw(
217 author,
218 issue.CreatedAt.Unix(),
219 text.CleanupOneLine(title), // TODO: this is the *current* title, not the original one
220 text.Cleanup(textInput),
221 nil,
222 map[string]string{
223 core.MetaKeyOrigin: target,
224 metaKeyGithubId: parseId(issue.Id),
225 metaKeyGithubUrl: issue.Url.String(),
226 })
227 if err != nil {
228 return nil, err
229 }
230 // importing a new bug
231 gi.out <- core.NewImportBug(b.Id())
232
233 return b, nil
234}
235
236func (gi *githubImporter) ensureIssueEdit(ctx context.Context, repo *cache.RepoCache, bug *cache.BugCache, ghIssueId githubv4.ID, edit *userContentEdit) error {
237 return gi.ensureCommentEdit(ctx, repo, bug, ghIssueId, edit)
238}
239
240func (gi *githubImporter) ensureTimelineItem(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, item *timelineItem) error {
241
242 switch item.Typename {
243 case "IssueComment":
244 err := gi.ensureComment(ctx, repo, b, &item.IssueComment, nil)
245 if err != nil {
246 return fmt.Errorf("timeline comment creation: %v", err)
247 }
248 return nil
249
250 case "LabeledEvent":
251 id := parseId(item.LabeledEvent.Id)
252 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
253 if err == nil {
254 return nil
255 }
256
257 if err != cache.ErrNoMatchingOp {
258 return err
259 }
260 author, err := gi.ensurePerson(ctx, repo, item.LabeledEvent.Actor)
261 if err != nil {
262 return err
263 }
264 op, err := b.ForceChangeLabelsRaw(
265 author,
266 item.LabeledEvent.CreatedAt.Unix(),
267 []string{
268 text.CleanupOneLine(string(item.LabeledEvent.Label.Name)),
269 },
270 nil,
271 map[string]string{metaKeyGithubId: id},
272 )
273 if err != nil {
274 return err
275 }
276
277 gi.out <- core.NewImportLabelChange(op.Id())
278 return nil
279
280 case "UnlabeledEvent":
281 id := parseId(item.UnlabeledEvent.Id)
282 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
283 if err == nil {
284 return nil
285 }
286 if err != cache.ErrNoMatchingOp {
287 return err
288 }
289 author, err := gi.ensurePerson(ctx, repo, item.UnlabeledEvent.Actor)
290 if err != nil {
291 return err
292 }
293
294 op, err := b.ForceChangeLabelsRaw(
295 author,
296 item.UnlabeledEvent.CreatedAt.Unix(),
297 nil,
298 []string{
299 text.CleanupOneLine(string(item.UnlabeledEvent.Label.Name)),
300 },
301 map[string]string{metaKeyGithubId: id},
302 )
303 if err != nil {
304 return err
305 }
306
307 gi.out <- core.NewImportLabelChange(op.Id())
308 return nil
309
310 case "ClosedEvent":
311 id := parseId(item.ClosedEvent.Id)
312 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
313 if err != cache.ErrNoMatchingOp {
314 return err
315 }
316 if err == nil {
317 return nil
318 }
319 author, err := gi.ensurePerson(ctx, repo, item.ClosedEvent.Actor)
320 if err != nil {
321 return err
322 }
323 op, err := b.CloseRaw(
324 author,
325 item.ClosedEvent.CreatedAt.Unix(),
326 map[string]string{metaKeyGithubId: id},
327 )
328
329 if err != nil {
330 return err
331 }
332
333 gi.out <- core.NewImportStatusChange(op.Id())
334 return nil
335
336 case "ReopenedEvent":
337 id := parseId(item.ReopenedEvent.Id)
338 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
339 if err != cache.ErrNoMatchingOp {
340 return err
341 }
342 if err == nil {
343 return nil
344 }
345 author, err := gi.ensurePerson(ctx, repo, item.ReopenedEvent.Actor)
346 if err != nil {
347 return err
348 }
349 op, err := b.OpenRaw(
350 author,
351 item.ReopenedEvent.CreatedAt.Unix(),
352 map[string]string{metaKeyGithubId: id},
353 )
354
355 if err != nil {
356 return err
357 }
358
359 gi.out <- core.NewImportStatusChange(op.Id())
360 return nil
361
362 case "RenamedTitleEvent":
363 id := parseId(item.RenamedTitleEvent.Id)
364 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
365 if err != cache.ErrNoMatchingOp {
366 return err
367 }
368 if err == nil {
369 return nil
370 }
371 author, err := gi.ensurePerson(ctx, repo, item.RenamedTitleEvent.Actor)
372 if err != nil {
373 return err
374 }
375
376 // At Github there exist issues with seemingly empty titles. An example is
377 // https://github.com/NixOS/nixpkgs/issues/72730 (here the title is actually
378 // a zero width space U+200B).
379 // Set title to some non-empty string, since git-bug does not accept empty titles.
380 title := text.CleanupOneLine(string(item.RenamedTitleEvent.CurrentTitle))
381 if text.Empty(title) {
382 title = EmptyTitlePlaceholder
383 }
384
385 op, err := b.SetTitleRaw(
386 author,
387 item.RenamedTitleEvent.CreatedAt.Unix(),
388 title,
389 map[string]string{metaKeyGithubId: id},
390 )
391 if err != nil {
392 return err
393 }
394
395 gi.out <- core.NewImportTitleEdition(op.Id())
396 return nil
397 }
398
399 return nil
400}
401
402func (gi *githubImporter) ensureCommentEdit(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, ghTargetId githubv4.ID, edit *userContentEdit) error {
403 // find comment
404 target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(ghTargetId))
405 if err != nil {
406 return err
407 }
408 _, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
409 if err == nil {
410 return nil
411 }
412 if err != cache.ErrNoMatchingOp {
413 // real error
414 return err
415 }
416
417 editor, err := gi.ensurePerson(ctx, repo, edit.Editor)
418 if err != nil {
419 return err
420 }
421
422 if edit.DeletedAt != nil {
423 // comment deletion, not supported yet
424 return nil
425 }
426
427 // comment edition
428 op, err := b.EditCommentRaw(
429 editor,
430 edit.CreatedAt.Unix(),
431 target,
432 text.Cleanup(string(*edit.Diff)),
433 map[string]string{
434 metaKeyGithubId: parseId(edit.Id),
435 },
436 )
437
438 if err != nil {
439 return err
440 }
441
442 gi.out <- core.NewImportCommentEdition(op.Id())
443 return nil
444}
445
446func (gi *githubImporter) ensureComment(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, comment *issueComment, firstEdit *userContentEdit) error {
447 author, err := gi.ensurePerson(ctx, repo, comment.Author)
448 if err != nil {
449 return err
450 }
451
452 _, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(comment.Id))
453 if err == nil {
454 return nil
455 }
456 if err != cache.ErrNoMatchingOp {
457 // real error
458 return err
459 }
460
461 var textInput string
462 if firstEdit != nil {
463 // use the first comment edit: it represents the comment creation itself
464 textInput = string(*firstEdit.Diff)
465 } else {
466 // if there are not comment edits, then the comment struct holds the comment creation
467 textInput = string(comment.Body)
468 }
469
470 // add comment operation
471 op, err := b.AddCommentRaw(
472 author,
473 comment.CreatedAt.Unix(),
474 text.Cleanup(textInput),
475 nil,
476 map[string]string{
477 metaKeyGithubId: parseId(comment.Id),
478 metaKeyGithubUrl: comment.Url.String(),
479 },
480 )
481 if err != nil {
482 return err
483 }
484
485 gi.out <- core.NewImportComment(op.Id())
486 return nil
487}
488
489// ensurePerson create a bug.Person from the Github data
490func (gi *githubImporter) ensurePerson(ctx context.Context, repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
491 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
492 // in it's UI. So we need a special case to get it.
493 if actor == nil {
494 return gi.getGhost(ctx, repo)
495 }
496
497 // Look first in the cache
498 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
499 if err == nil {
500 return i, nil
501 }
502 if entity.IsErrMultipleMatch(err) {
503 return nil, err
504 }
505
506 // importing a new identity
507 var name string
508 var email string
509
510 switch actor.Typename {
511 case "User":
512 if actor.User.Name != nil {
513 name = string(*(actor.User.Name))
514 }
515 email = string(actor.User.Email)
516 case "Organization":
517 if actor.Organization.Name != nil {
518 name = string(*(actor.Organization.Name))
519 }
520 if actor.Organization.Email != nil {
521 email = string(*(actor.Organization.Email))
522 }
523 case "Bot":
524 }
525
526 // Name is not necessarily set, fallback to login as a name is required in the identity
527 if name == "" {
528 name = string(actor.Login)
529 }
530
531 i, err = repo.NewIdentityRaw(
532 name,
533 email,
534 string(actor.Login),
535 string(actor.AvatarUrl),
536 nil,
537 map[string]string{
538 metaKeyGithubLogin: string(actor.Login),
539 },
540 )
541
542 if err != nil {
543 return nil, err
544 }
545
546 gi.out <- core.NewImportIdentity(i.Id())
547 return i, nil
548}
549
550func (gi *githubImporter) getGhost(ctx context.Context, repo *cache.RepoCache) (*cache.IdentityCache, error) {
551 loginName := "ghost"
552 // Look first in the cache
553 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, loginName)
554 if err == nil {
555 return i, nil
556 }
557 if entity.IsErrMultipleMatch(err) {
558 return nil, err
559 }
560 user, err := gi.mediator.User(ctx, loginName)
561 if err != nil {
562 return nil, err
563 }
564 userName := ""
565 if user.Name != nil {
566 userName = string(*user.Name)
567 }
568 return repo.NewIdentityRaw(
569 userName,
570 "",
571 string(user.Login),
572 string(user.AvatarUrl),
573 nil,
574 map[string]string{
575 metaKeyGithubLogin: string(user.Login),
576 },
577 )
578}
579
580// parseId converts the unusable githubv4.ID (an interface{}) into a string
581func parseId(id githubv4.ID) string {
582 return fmt.Sprintf("%v", id)
583}