1package github
2
3import (
4 "context"
5 "fmt"
6 "time"
7
8 "github.com/shurcooL/githubv4"
9
10 "github.com/MichaelMure/git-bug/bridge/core"
11 "github.com/MichaelMure/git-bug/bridge/core/auth"
12 "github.com/MichaelMure/git-bug/bug"
13 "github.com/MichaelMure/git-bug/cache"
14 "github.com/MichaelMure/git-bug/entity"
15 "github.com/MichaelMure/git-bug/util/text"
16)
17
18const EMPTY_TITLE_PLACEHOLDER = "<empty string>"
19
20// githubImporter implement the Importer interface
21type githubImporter struct {
22 conf core.Configuration
23
24 // mediator to access the Github API
25 mediator *importMediator
26
27 // send only channel
28 out chan<- core.ImportResult
29}
30
31func (gi *githubImporter) Init(_ context.Context, _ *cache.RepoCache, conf core.Configuration) error {
32 gi.conf = conf
33 return nil
34}
35
36// ImportAll iterate over all the configured repository issues and ensure the creation of the
37// missing issues / timeline items / edits / label events ...
38func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
39 creds, err := auth.List(repo,
40 auth.WithTarget(target),
41 auth.WithKind(auth.KindToken),
42 auth.WithMeta(auth.MetaKeyLogin, gi.conf[confKeyDefaultLogin]),
43 )
44 if err != nil {
45 return nil, err
46 }
47 if len(creds) <= 0 {
48 return nil, ErrMissingIdentityToken
49 }
50 client := buildClient(creds[0].(*auth.Token))
51 gi.mediator = NewImportMediator(ctx, client, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
52 out := make(chan core.ImportResult)
53 gi.out = out
54
55 go func() {
56 defer close(gi.out)
57
58 // Loop over all matching issues
59 for event := range gi.mediator.Issues {
60 var issue issue
61 var issueEdits <-chan userContentEditEvent
62 var timelineItems <-chan timelineEvent
63 switch e := event.(type) {
64 case messageEvent:
65 fmt.Println(e.msg)
66 continue
67 case issueData:
68 issue = e.issue
69 issueEdits = e.issueEdits
70 timelineItems = e.timelineItems
71 default:
72 panic(fmt.Sprint("Unknown event type"))
73 }
74 // create issue
75 b, err := gi.ensureIssue(ctx, repo, &issue, issueEdits)
76 if err != nil {
77 err := fmt.Errorf("issue creation: %v", err)
78 out <- core.NewImportError(err, "")
79 return
80 }
81
82 // loop over timeline items
83 for event := range timelineItems {
84 var item timelineItem
85 var edits <-chan userContentEditEvent
86 switch e := event.(type) {
87 case messageEvent:
88 fmt.Println(e.msg)
89 continue
90 case timelineData:
91 item = e.timelineItem
92 edits = e.userContentEdits
93 default:
94 panic(fmt.Sprint("Unknown event type"))
95 }
96 err := gi.ensureTimelineItem(ctx, repo, b, &item, edits)
97 if err != nil {
98 err = fmt.Errorf("timeline item creation: %v", err)
99 out <- core.NewImportError(err, "")
100 return
101 }
102 }
103
104 if !b.NeedCommit() {
105 out <- core.NewImportNothing(b.Id(), "no imported operation")
106 } else if err := b.Commit(); err != nil {
107 // commit bug state
108 err = fmt.Errorf("bug commit: %v", err)
109 out <- core.NewImportError(err, "")
110 return
111 }
112 }
113
114 if err := gi.mediator.Error(); err != nil {
115 gi.out <- core.NewImportError(err, "")
116 }
117 }()
118
119 return out, nil
120}
121
122// getNextUserContentEdit reads the input channel, handles messages, and returns the next
123// userContentEditData.
124func getNextUserContentEdit(in <-chan userContentEditEvent) (*userContentEditData, bool) {
125 for {
126 event, hasEvent := <-in
127 if !hasEvent {
128 return nil, false
129 }
130 switch e := event.(type) {
131 case messageEvent:
132 fmt.Println(e.msg)
133 continue
134 case userContentEditData:
135 return &e, true
136 default:
137 panic(fmt.Sprint("Unknown event type"))
138 }
139 }
140}
141
142func (gi *githubImporter) ensureIssue(ctx context.Context, repo *cache.RepoCache, issue *issue, issueEditEvents <-chan userContentEditEvent) (*cache.BugCache, error) {
143 author, err := gi.ensurePerson(ctx, repo, issue.Author)
144 if err != nil {
145 return nil, err
146 }
147
148 // resolve bug
149 b, err := repo.ResolveBugMatcher(func(excerpt *cache.BugExcerpt) bool {
150 return excerpt.CreateMetadata[core.MetaKeyOrigin] == target &&
151 excerpt.CreateMetadata[metaKeyGithubId] == parseId(issue.Id)
152 })
153 if err != nil && err != bug.ErrBugNotExist {
154 return nil, err
155 }
156
157 // get first issue edit
158 // if it exists, then it holds the bug creation
159 firstEdit, hasEdit := getNextUserContentEdit(issueEditEvents)
160
161 // At Github there exist issues with seemingly empty titles. An example is
162 // https://github.com/NixOS/nixpkgs/issues/72730 .
163 // The title provided by the GraphQL API actually consists of a space followed by a
164 // zero width space (U+200B). This title would cause the NewBugRaw() function to
165 // return an error: empty title.
166 title := string(issue.Title)
167 if title == " \u200b" { // U+200B == zero width space
168 title = EMPTY_TITLE_PLACEHOLDER
169 }
170
171 if err == bug.ErrBugNotExist {
172 var textInput string
173 if hasEdit {
174 // use the first issue edit: it represents the bug creation itself
175 textInput = string(*firstEdit.Diff)
176 } else {
177 // if there are no issue edits then the issue struct holds the bug creation
178 textInput = string(issue.Body)
179 }
180 cleanText, err := text.Cleanup(textInput)
181 if err != nil {
182 return nil, err
183 }
184 // create bug
185 b, _, err = repo.NewBugRaw(
186 author,
187 issue.CreatedAt.Unix(),
188 title, // TODO: this is the *current* title, not the original one
189 cleanText,
190 nil,
191 map[string]string{
192 core.MetaKeyOrigin: target,
193 metaKeyGithubId: parseId(issue.Id),
194 metaKeyGithubUrl: issue.Url.String(),
195 })
196 if err != nil {
197 return nil, err
198 }
199 // importing a new bug
200 gi.out <- core.NewImportBug(b.Id())
201 }
202 if b == nil {
203 return nil, fmt.Errorf("finding or creating issue")
204 }
205 // process remaining issue edits, if they exist
206 for {
207 edit, hasEdit := getNextUserContentEdit(issueEditEvents)
208 if !hasEdit {
209 break
210 }
211 // other edits will be added as CommentEdit operations
212 target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(issue.Id))
213 if err == cache.ErrNoMatchingOp {
214 // original comment is missing somehow, issuing a warning
215 gi.out <- core.NewImportWarning(fmt.Errorf("comment ID %s to edit is missing", parseId(issue.Id)), b.Id())
216 continue
217 }
218 if err != nil {
219 return nil, err
220 }
221
222 err = gi.ensureCommentEdit(ctx, repo, b, target, &edit.userContentEdit)
223 if err != nil {
224 return nil, err
225 }
226 }
227 return b, nil
228}
229
230func (gi *githubImporter) ensureTimelineItem(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, item *timelineItem, commentEdits <-chan userContentEditEvent) error {
231
232 switch item.Typename {
233 case "IssueComment":
234 err := gi.ensureComment(ctx, repo, b, &item.IssueComment, commentEdits)
235 if err != nil {
236 return fmt.Errorf("timeline comment creation: %v", err)
237 }
238 return nil
239
240 case "LabeledEvent":
241 id := parseId(item.LabeledEvent.Id)
242 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
243 if err == nil {
244 return nil
245 }
246
247 if err != cache.ErrNoMatchingOp {
248 return err
249 }
250 author, err := gi.ensurePerson(ctx, repo, item.LabeledEvent.Actor)
251 if err != nil {
252 return err
253 }
254 op, err := b.ForceChangeLabelsRaw(
255 author,
256 item.LabeledEvent.CreatedAt.Unix(),
257 []string{
258 string(item.LabeledEvent.Label.Name),
259 },
260 nil,
261 map[string]string{metaKeyGithubId: id},
262 )
263 if err != nil {
264 return err
265 }
266
267 gi.out <- core.NewImportLabelChange(op.Id())
268 return nil
269
270 case "UnlabeledEvent":
271 id := parseId(item.UnlabeledEvent.Id)
272 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
273 if err == nil {
274 return nil
275 }
276 if err != cache.ErrNoMatchingOp {
277 return err
278 }
279 author, err := gi.ensurePerson(ctx, repo, item.UnlabeledEvent.Actor)
280 if err != nil {
281 return err
282 }
283
284 op, err := b.ForceChangeLabelsRaw(
285 author,
286 item.UnlabeledEvent.CreatedAt.Unix(),
287 nil,
288 []string{
289 string(item.UnlabeledEvent.Label.Name),
290 },
291 map[string]string{metaKeyGithubId: id},
292 )
293 if err != nil {
294 return err
295 }
296
297 gi.out <- core.NewImportLabelChange(op.Id())
298 return nil
299
300 case "ClosedEvent":
301 id := parseId(item.ClosedEvent.Id)
302 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
303 if err != cache.ErrNoMatchingOp {
304 return err
305 }
306 if err == nil {
307 return nil
308 }
309 author, err := gi.ensurePerson(ctx, repo, item.ClosedEvent.Actor)
310 if err != nil {
311 return err
312 }
313 op, err := b.CloseRaw(
314 author,
315 item.ClosedEvent.CreatedAt.Unix(),
316 map[string]string{metaKeyGithubId: id},
317 )
318
319 if err != nil {
320 return err
321 }
322
323 gi.out <- core.NewImportStatusChange(op.Id())
324 return nil
325
326 case "ReopenedEvent":
327 id := parseId(item.ReopenedEvent.Id)
328 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
329 if err != cache.ErrNoMatchingOp {
330 return err
331 }
332 if err == nil {
333 return nil
334 }
335 author, err := gi.ensurePerson(ctx, repo, item.ReopenedEvent.Actor)
336 if err != nil {
337 return err
338 }
339 op, err := b.OpenRaw(
340 author,
341 item.ReopenedEvent.CreatedAt.Unix(),
342 map[string]string{metaKeyGithubId: id},
343 )
344
345 if err != nil {
346 return err
347 }
348
349 gi.out <- core.NewImportStatusChange(op.Id())
350 return nil
351
352 case "RenamedTitleEvent":
353 id := parseId(item.RenamedTitleEvent.Id)
354 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
355 if err != cache.ErrNoMatchingOp {
356 return err
357 }
358 if err == nil {
359 return nil
360 }
361 author, err := gi.ensurePerson(ctx, repo, item.RenamedTitleEvent.Actor)
362 if err != nil {
363 return err
364 }
365
366 // At Github there exist issues with seemingly empty titles. An example is
367 // https://github.com/NixOS/nixpkgs/issues/72730 .
368 // The title provided by the GraphQL API actually consists of a space followed
369 // by a zero width space (U+200B). This title would cause the NewBugRaw()
370 // function to return an error: empty title.
371 title := string(item.RenamedTitleEvent.CurrentTitle)
372 if title == " \u200b" { // U+200B == zero width space
373 title = EMPTY_TITLE_PLACEHOLDER
374 }
375
376 op, err := b.SetTitleRaw(
377 author,
378 item.RenamedTitleEvent.CreatedAt.Unix(),
379 title,
380 map[string]string{metaKeyGithubId: id},
381 )
382 if err != nil {
383 return err
384 }
385
386 gi.out <- core.NewImportTitleEdition(op.Id())
387 return nil
388 }
389
390 return nil
391}
392
393func (gi *githubImporter) ensureComment(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, comment *issueComment, commentEditEvents <-chan userContentEditEvent) error {
394 author, err := gi.ensurePerson(ctx, repo, comment.Author)
395 if err != nil {
396 return err
397 }
398
399 targetOpID, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(comment.Id))
400 if err != nil && err != cache.ErrNoMatchingOp {
401 // real error
402 return err
403 }
404 firstEdit, hasEdit := getNextUserContentEdit(commentEditEvents)
405 if err == cache.ErrNoMatchingOp {
406 var textInput string
407 if hasEdit {
408 // use the first comment edit: it represents the comment creation itself
409 textInput = string(*firstEdit.Diff)
410 } else {
411 // if there are not comment edits, then the comment struct holds the comment creation
412 textInput = string(comment.Body)
413 }
414 cleanText, err := text.Cleanup(textInput)
415 if err != nil {
416 return err
417 }
418
419 // add comment operation
420 op, err := b.AddCommentRaw(
421 author,
422 comment.CreatedAt.Unix(),
423 cleanText,
424 nil,
425 map[string]string{
426 metaKeyGithubId: parseId(comment.Id),
427 metaKeyGithubUrl: comment.Url.String(),
428 },
429 )
430 if err != nil {
431 return err
432 }
433
434 gi.out <- core.NewImportComment(op.Id())
435 targetOpID = op.Id()
436 }
437 if targetOpID == "" {
438 return fmt.Errorf("finding or creating issue comment")
439 }
440 // process remaining comment edits, if they exist
441 for {
442 edit, hasEdit := getNextUserContentEdit(commentEditEvents)
443 if !hasEdit {
444 break
445 }
446 // ensure editor identity
447 _, err := gi.ensurePerson(ctx, repo, edit.Editor)
448 if err != nil {
449 return err
450 }
451
452 err = gi.ensureCommentEdit(ctx, repo, b, targetOpID, &edit.userContentEdit)
453 if err != nil {
454 return err
455 }
456 }
457 return nil
458}
459
460func (gi *githubImporter) ensureCommentEdit(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, target entity.Id, edit *userContentEdit) error {
461 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
462 if err == nil {
463 return nil
464 }
465 if err != cache.ErrNoMatchingOp {
466 // real error
467 return err
468 }
469
470 editor, err := gi.ensurePerson(ctx, repo, edit.Editor)
471 if err != nil {
472 return err
473 }
474
475 switch {
476 case edit.DeletedAt != nil:
477 // comment deletion, not supported yet
478 return nil
479
480 case edit.DeletedAt == nil:
481
482 cleanText, err := text.Cleanup(string(*edit.Diff))
483 if err != nil {
484 return err
485 }
486
487 // comment edition
488 op, err := b.EditCommentRaw(
489 editor,
490 edit.CreatedAt.Unix(),
491 target,
492 cleanText,
493 map[string]string{
494 metaKeyGithubId: parseId(edit.Id),
495 },
496 )
497
498 if err != nil {
499 return err
500 }
501
502 gi.out <- core.NewImportCommentEdition(op.Id())
503 return nil
504 }
505 return nil
506}
507
508// ensurePerson create a bug.Person from the Github data
509func (gi *githubImporter) ensurePerson(ctx context.Context, repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
510 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
511 // in it's UI. So we need a special case to get it.
512 if actor == nil {
513 return gi.getGhost(ctx, repo)
514 }
515
516 // Look first in the cache
517 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
518 if err == nil {
519 return i, nil
520 }
521 if entity.IsErrMultipleMatch(err) {
522 return nil, err
523 }
524
525 // importing a new identity
526 var name string
527 var email string
528
529 switch actor.Typename {
530 case "User":
531 if actor.User.Name != nil {
532 name = string(*(actor.User.Name))
533 }
534 email = string(actor.User.Email)
535 case "Organization":
536 if actor.Organization.Name != nil {
537 name = string(*(actor.Organization.Name))
538 }
539 if actor.Organization.Email != nil {
540 email = string(*(actor.Organization.Email))
541 }
542 case "Bot":
543 }
544
545 // Name is not necessarily set, fallback to login as a name is required in the identity
546 if name == "" {
547 name = string(actor.Login)
548 }
549
550 i, err = repo.NewIdentityRaw(
551 name,
552 email,
553 string(actor.Login),
554 string(actor.AvatarUrl),
555 map[string]string{
556 metaKeyGithubLogin: string(actor.Login),
557 },
558 )
559
560 if err != nil {
561 return nil, err
562 }
563
564 gi.out <- core.NewImportIdentity(i.Id())
565 return i, nil
566}
567
568func (gi *githubImporter) getGhost(ctx context.Context, repo *cache.RepoCache) (*cache.IdentityCache, error) {
569 loginName := "ghost"
570 // Look first in the cache
571 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, loginName)
572 if err == nil {
573 return i, nil
574 }
575 if entity.IsErrMultipleMatch(err) {
576 return nil, err
577 }
578 user, err := gi.mediator.User(ctx, loginName)
579 userName := ""
580 if user.Name != nil {
581 userName = string(*user.Name)
582 }
583 return repo.NewIdentityRaw(
584 userName,
585 "",
586 string(user.Login),
587 string(user.AvatarUrl),
588 map[string]string{
589 metaKeyGithubLogin: string(user.Login),
590 },
591 )
592}
593
594// parseId converts the unusable githubv4.ID (an interface{}) into a string
595func parseId(id githubv4.ID) string {
596 return fmt.Sprintf("%v", id)
597}