1package github
2
3import (
4 "context"
5 "fmt"
6 "time"
7
8 "github.com/shurcooL/githubv4"
9
10 "github.com/MichaelMure/git-bug/bridge/core"
11 "github.com/MichaelMure/git-bug/bridge/core/auth"
12 "github.com/MichaelMure/git-bug/cache"
13 "github.com/MichaelMure/git-bug/entities/bug"
14 "github.com/MichaelMure/git-bug/entity"
15 "github.com/MichaelMure/git-bug/util/text"
16)
17
18const EmptyTitlePlaceholder = "<empty string>"
19
20// githubImporter implement the Importer interface
21type githubImporter struct {
22 conf core.Configuration
23
24 // default client
25 client *rateLimitHandlerClient
26
27 // mediator to access the Github API
28 mediator *importMediator
29
30 // send only channel
31 out chan<- core.ImportResult
32}
33
34func (gi *githubImporter) Init(_ context.Context, repo *cache.RepoCache, conf core.Configuration) error {
35 gi.conf = conf
36 creds, err := auth.List(repo,
37 auth.WithTarget(target),
38 auth.WithKind(auth.KindToken),
39 auth.WithMeta(auth.MetaKeyLogin, conf[confKeyDefaultLogin]),
40 )
41 if err != nil {
42 return err
43 }
44 if len(creds) <= 0 {
45 return ErrMissingIdentityToken
46 }
47 gi.client = buildClient(creds[0].(*auth.Token))
48
49 return nil
50}
51
52// ImportAll iterate over all the configured repository issues and ensure the creation of the
53// missing issues / timeline items / edits / label events ...
54func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
55 gi.mediator = NewImportMediator(ctx, gi.client, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
56 out := make(chan core.ImportResult)
57 gi.out = out
58
59 go func() {
60 defer close(gi.out)
61 var currBug *cache.BugCache
62 var currEvent ImportEvent
63 var nextEvent ImportEvent
64 var err error
65 for {
66 // An IssueEvent contains the issue in its most recent state. If an issue
67 // has at least one issue edit, then the history of the issue edits is
68 // represented by IssueEditEvents. That is, the unedited (original) issue
69 // might be saved only in the IssueEditEvent following the IssueEvent.
70 // Since we replicate the edit history we need to either use the IssueEvent
71 // (if there are no edits) or the IssueEvent together with its first
72 // IssueEditEvent (if there are edits).
73 // Exactly the same is true for comments and comment edits.
74 // As a consequence we need to look at the current event and one look ahead
75 // event.
76
77 currEvent = nextEvent
78 if currEvent == nil {
79 currEvent = gi.getEventHandleMsgs()
80 }
81 if currEvent == nil {
82 break
83 }
84 nextEvent = gi.getEventHandleMsgs()
85
86 switch event := currEvent.(type) {
87 case RateLimitingEvent:
88 out <- core.NewImportRateLimiting(event.msg)
89 case IssueEvent:
90 // first: commit what is being held in currBug
91 if err = gi.commit(currBug, out); err != nil {
92 out <- core.NewImportError(err, "")
93 return
94 }
95 // second: create new issue
96 switch next := nextEvent.(type) {
97 case IssueEditEvent:
98 // consuming and using next event
99 nextEvent = nil
100 currBug, err = gi.ensureIssue(ctx, repo, &event.issue, &next.userContentEdit)
101 default:
102 currBug, err = gi.ensureIssue(ctx, repo, &event.issue, nil)
103 }
104 if err != nil {
105 err := fmt.Errorf("issue creation: %v", err)
106 out <- core.NewImportError(err, "")
107 return
108 }
109 case IssueEditEvent:
110 err = gi.ensureIssueEdit(ctx, repo, currBug, event.issueId, &event.userContentEdit)
111 if err != nil {
112 err = fmt.Errorf("issue edit: %v", err)
113 out <- core.NewImportError(err, "")
114 return
115 }
116 case TimelineEvent:
117 if next, ok := nextEvent.(CommentEditEvent); ok && event.Typename == "IssueComment" {
118 // consuming and using next event
119 nextEvent = nil
120 err = gi.ensureComment(ctx, repo, currBug, &event.timelineItem.IssueComment, &next.userContentEdit)
121 } else {
122 err = gi.ensureTimelineItem(ctx, repo, currBug, &event.timelineItem)
123 }
124 if err != nil {
125 err = fmt.Errorf("timeline item creation: %v", err)
126 out <- core.NewImportError(err, "")
127 return
128 }
129 case CommentEditEvent:
130 err = gi.ensureCommentEdit(ctx, repo, currBug, event.commentId, &event.userContentEdit)
131 if err != nil {
132 err = fmt.Errorf("comment edit: %v", err)
133 out <- core.NewImportError(err, "")
134 return
135 }
136 default:
137 panic("Unknown event type")
138 }
139 }
140 // commit what is being held in currBug before returning
141 if err = gi.commit(currBug, out); err != nil {
142 out <- core.NewImportError(err, "")
143 }
144 if err = gi.mediator.Error(); err != nil {
145 gi.out <- core.NewImportError(err, "")
146 }
147 }()
148
149 return out, nil
150}
151
152func (gi *githubImporter) getEventHandleMsgs() ImportEvent {
153 for {
154 // read event from import mediator
155 event := gi.mediator.NextImportEvent()
156 // consume (and use) all rate limiting events
157 if e, ok := event.(RateLimitingEvent); ok {
158 gi.out <- core.NewImportRateLimiting(e.msg)
159 continue
160 }
161 return event
162 }
163}
164
165func (gi *githubImporter) commit(b *cache.BugCache, out chan<- core.ImportResult) error {
166 if b == nil {
167 return nil
168 }
169 if !b.NeedCommit() {
170 out <- core.NewImportNothing(b.Id(), "no imported operation")
171 return nil
172 } else if err := b.Commit(); err != nil {
173 // commit bug state
174 return fmt.Errorf("bug commit: %v", err)
175 }
176 return nil
177}
178
179func (gi *githubImporter) ensureIssue(ctx context.Context, repo *cache.RepoCache, issue *issue, issueEdit *userContentEdit) (*cache.BugCache, error) {
180 author, err := gi.ensurePerson(ctx, repo, issue.Author)
181 if err != nil {
182 return nil, err
183 }
184
185 // resolve bug
186 b, err := repo.ResolveBugMatcher(func(excerpt *cache.BugExcerpt) bool {
187 return excerpt.CreateMetadata[metaKeyGithubUrl] == issue.Url.String() &&
188 excerpt.CreateMetadata[metaKeyGithubId] == parseId(issue.Id)
189 })
190 if err == nil {
191 return b, nil
192 }
193 if err != bug.ErrBugNotExist {
194 return nil, err
195 }
196
197 // At Github there exist issues with seemingly empty titles. An example is
198 // https://github.com/NixOS/nixpkgs/issues/72730 (here the title is actually
199 // a zero width space U+200B).
200 // Set title to some non-empty string, since git-bug does not accept empty titles.
201 title := text.CleanupOneLine(string(issue.Title))
202 if text.Empty(title) {
203 title = EmptyTitlePlaceholder
204 }
205
206 var textInput string
207 if issueEdit != nil {
208 // use the first issue edit: it represents the bug creation itself
209 textInput = string(*issueEdit.Diff)
210 } else {
211 // if there are no issue edits then the issue struct holds the bug creation
212 textInput = string(issue.Body)
213 }
214
215 // create bug
216 b, _, err = repo.NewBugRaw(
217 author,
218 issue.CreatedAt.Unix(),
219 text.CleanupOneLine(title), // TODO: this is the *current* title, not the original one
220 text.Cleanup(textInput),
221 nil,
222 map[string]string{
223 core.MetaKeyOrigin: target,
224 metaKeyGithubId: parseId(issue.Id),
225 metaKeyGithubUrl: issue.Url.String(),
226 })
227 if err != nil {
228 return nil, err
229 }
230 // importing a new bug
231 gi.out <- core.NewImportBug(b.Id())
232
233 return b, nil
234}
235
236func (gi *githubImporter) ensureIssueEdit(ctx context.Context, repo *cache.RepoCache, bug *cache.BugCache, ghIssueId githubv4.ID, edit *userContentEdit) error {
237 return gi.ensureCommentEdit(ctx, repo, bug, ghIssueId, edit)
238}
239
240func (gi *githubImporter) ensureTimelineItem(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, item *timelineItem) error {
241
242 switch item.Typename {
243 case "IssueComment":
244 err := gi.ensureComment(ctx, repo, b, &item.IssueComment, nil)
245 if err != nil {
246 return fmt.Errorf("timeline comment creation: %v", err)
247 }
248 return nil
249
250 case "LabeledEvent":
251 id := parseId(item.LabeledEvent.Id)
252 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
253 if err == nil {
254 return nil
255 }
256
257 if err != cache.ErrNoMatchingOp {
258 return err
259 }
260 author, err := gi.ensurePerson(ctx, repo, item.LabeledEvent.Actor)
261 if err != nil {
262 return err
263 }
264 op, err := b.ForceChangeLabelsRaw(
265 author,
266 item.LabeledEvent.CreatedAt.Unix(),
267 []string{
268 text.CleanupOneLine(string(item.LabeledEvent.Label.Name)),
269 },
270 nil,
271 map[string]string{metaKeyGithubId: id},
272 )
273 if err != nil {
274 return err
275 }
276
277 gi.out <- core.NewImportLabelChange(op.Id())
278 return nil
279
280 case "UnlabeledEvent":
281 id := parseId(item.UnlabeledEvent.Id)
282 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
283 if err == nil {
284 return nil
285 }
286 if err != cache.ErrNoMatchingOp {
287 return err
288 }
289 author, err := gi.ensurePerson(ctx, repo, item.UnlabeledEvent.Actor)
290 if err != nil {
291 return err
292 }
293
294 op, err := b.ForceChangeLabelsRaw(
295 author,
296 item.UnlabeledEvent.CreatedAt.Unix(),
297 nil,
298 []string{
299 text.CleanupOneLine(string(item.UnlabeledEvent.Label.Name)),
300 },
301 map[string]string{metaKeyGithubId: id},
302 )
303 if err != nil {
304 return err
305 }
306
307 gi.out <- core.NewImportLabelChange(op.Id())
308 return nil
309
310 case "ClosedEvent":
311 id := parseId(item.ClosedEvent.Id)
312 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
313 if err != cache.ErrNoMatchingOp {
314 return err
315 }
316 if err == nil {
317 return nil
318 }
319 author, err := gi.ensurePerson(ctx, repo, item.ClosedEvent.Actor)
320 if err != nil {
321 return err
322 }
323 op, err := b.CloseRaw(
324 author,
325 item.ClosedEvent.CreatedAt.Unix(),
326 map[string]string{metaKeyGithubId: id},
327 )
328
329 if err != nil {
330 return err
331 }
332
333 gi.out <- core.NewImportStatusChange(op.Id())
334 return nil
335
336 case "ReopenedEvent":
337 id := parseId(item.ReopenedEvent.Id)
338 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
339 if err != cache.ErrNoMatchingOp {
340 return err
341 }
342 if err == nil {
343 return nil
344 }
345 author, err := gi.ensurePerson(ctx, repo, item.ReopenedEvent.Actor)
346 if err != nil {
347 return err
348 }
349 op, err := b.OpenRaw(
350 author,
351 item.ReopenedEvent.CreatedAt.Unix(),
352 map[string]string{metaKeyGithubId: id},
353 )
354
355 if err != nil {
356 return err
357 }
358
359 gi.out <- core.NewImportStatusChange(op.Id())
360 return nil
361
362 case "RenamedTitleEvent":
363 id := parseId(item.RenamedTitleEvent.Id)
364 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
365 if err != cache.ErrNoMatchingOp {
366 return err
367 }
368 if err == nil {
369 return nil
370 }
371 author, err := gi.ensurePerson(ctx, repo, item.RenamedTitleEvent.Actor)
372 if err != nil {
373 return err
374 }
375
376 // At Github there exist issues with seemingly empty titles. An example is
377 // https://github.com/NixOS/nixpkgs/issues/72730 (here the title is actually
378 // a zero width space U+200B).
379 // Set title to some non-empty string, since git-bug does not accept empty titles.
380 title := text.CleanupOneLine(string(item.RenamedTitleEvent.CurrentTitle))
381 if text.Empty(title) {
382 title = EmptyTitlePlaceholder
383 }
384
385 op, err := b.SetTitleRaw(
386 author,
387 item.RenamedTitleEvent.CreatedAt.Unix(),
388 title,
389 map[string]string{metaKeyGithubId: id},
390 )
391 if err != nil {
392 return err
393 }
394
395 gi.out <- core.NewImportTitleEdition(op.Id())
396 return nil
397 }
398
399 return nil
400}
401
402func (gi *githubImporter) ensureCommentEdit(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, ghTargetId githubv4.ID, edit *userContentEdit) error {
403 // find comment
404 target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(ghTargetId))
405 if err != nil {
406 return err
407 }
408 // check if the comment edition already exist
409 _, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
410 if err == nil {
411 return nil
412 }
413 if err != cache.ErrNoMatchingOp {
414 // real error
415 return err
416 }
417
418 editor, err := gi.ensurePerson(ctx, repo, edit.Editor)
419 if err != nil {
420 return err
421 }
422
423 if edit.DeletedAt != nil {
424 // comment deletion, not supported yet
425 return nil
426 }
427
428 // comment edition
429 op, err := b.EditCommentRaw(
430 editor,
431 edit.CreatedAt.Unix(),
432 entity.CombineIds(b.Id(), target),
433 text.Cleanup(string(*edit.Diff)),
434 map[string]string{
435 metaKeyGithubId: parseId(edit.Id),
436 },
437 )
438
439 if err != nil {
440 return err
441 }
442
443 gi.out <- core.NewImportCommentEdition(op.Id())
444 return nil
445}
446
447func (gi *githubImporter) ensureComment(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, comment *issueComment, firstEdit *userContentEdit) error {
448 author, err := gi.ensurePerson(ctx, repo, comment.Author)
449 if err != nil {
450 return err
451 }
452
453 _, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(comment.Id))
454 if err == nil {
455 return nil
456 }
457 if err != cache.ErrNoMatchingOp {
458 // real error
459 return err
460 }
461
462 var textInput string
463 if firstEdit != nil {
464 // use the first comment edit: it represents the comment creation itself
465 textInput = string(*firstEdit.Diff)
466 } else {
467 // if there are not comment edits, then the comment struct holds the comment creation
468 textInput = string(comment.Body)
469 }
470
471 // add comment operation
472 op, err := b.AddCommentRaw(
473 author,
474 comment.CreatedAt.Unix(),
475 text.Cleanup(textInput),
476 nil,
477 map[string]string{
478 metaKeyGithubId: parseId(comment.Id),
479 metaKeyGithubUrl: comment.Url.String(),
480 },
481 )
482 if err != nil {
483 return err
484 }
485
486 gi.out <- core.NewImportComment(op.Id())
487 return nil
488}
489
490// ensurePerson create a bug.Person from the Github data
491func (gi *githubImporter) ensurePerson(ctx context.Context, repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
492 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
493 // in it's UI. So we need a special case to get it.
494 if actor == nil {
495 return gi.getGhost(ctx, repo)
496 }
497
498 // Look first in the cache
499 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
500 if err == nil {
501 return i, nil
502 }
503 if entity.IsErrMultipleMatch(err) {
504 return nil, err
505 }
506
507 // importing a new identity
508 var name string
509 var email string
510
511 switch actor.Typename {
512 case "User":
513 if actor.User.Name != nil {
514 name = string(*(actor.User.Name))
515 }
516 email = string(actor.User.Email)
517 case "Organization":
518 if actor.Organization.Name != nil {
519 name = string(*(actor.Organization.Name))
520 }
521 if actor.Organization.Email != nil {
522 email = string(*(actor.Organization.Email))
523 }
524 case "Bot":
525 }
526
527 // Name is not necessarily set, fallback to login as a name is required in the identity
528 if name == "" {
529 name = string(actor.Login)
530 }
531
532 i, err = repo.NewIdentityRaw(
533 name,
534 email,
535 string(actor.Login),
536 string(actor.AvatarUrl),
537 nil,
538 map[string]string{
539 metaKeyGithubLogin: string(actor.Login),
540 },
541 )
542
543 if err != nil {
544 return nil, err
545 }
546
547 gi.out <- core.NewImportIdentity(i.Id())
548 return i, nil
549}
550
551func (gi *githubImporter) getGhost(ctx context.Context, repo *cache.RepoCache) (*cache.IdentityCache, error) {
552 loginName := "ghost"
553 // Look first in the cache
554 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, loginName)
555 if err == nil {
556 return i, nil
557 }
558 if entity.IsErrMultipleMatch(err) {
559 return nil, err
560 }
561 user, err := gi.mediator.User(ctx, loginName)
562 if err != nil {
563 return nil, err
564 }
565 userName := ""
566 if user.Name != nil {
567 userName = string(*user.Name)
568 }
569 return repo.NewIdentityRaw(
570 userName,
571 "",
572 string(user.Login),
573 string(user.AvatarUrl),
574 nil,
575 map[string]string{
576 metaKeyGithubLogin: string(user.Login),
577 },
578 )
579}
580
581// parseId converts the unusable githubv4.ID (an interface{}) into a string
582func parseId(id githubv4.ID) string {
583 return fmt.Sprintf("%v", id)
584}