1package github
2
3import (
4 "context"
5 "fmt"
6 "time"
7
8 "github.com/shurcooL/githubv4"
9
10 "github.com/MichaelMure/git-bug/bridge/core"
11 "github.com/MichaelMure/git-bug/bridge/core/auth"
12 "github.com/MichaelMure/git-bug/bug"
13 "github.com/MichaelMure/git-bug/cache"
14 "github.com/MichaelMure/git-bug/entity"
15 "github.com/MichaelMure/git-bug/util/text"
16)
17
18const EMPTY_TITLE_PLACEHOLDER = "<empty string>"
19
20// githubImporter implement the Importer interface
21type githubImporter struct {
22 conf core.Configuration
23
24 // mediator to access the Github API
25 mediator *importMediator
26
27 // send only channel
28 out chan<- core.ImportResult
29}
30
31func (gi *githubImporter) Init(_ context.Context, _ *cache.RepoCache, conf core.Configuration) error {
32 gi.conf = conf
33 return nil
34}
35
36// ImportAll iterate over all the configured repository issues and ensure the creation of the
37// missing issues / timeline items / edits / label events ...
38func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
39 creds, err := auth.List(repo,
40 auth.WithTarget(target),
41 auth.WithKind(auth.KindToken),
42 auth.WithMeta(auth.MetaKeyLogin, gi.conf[confKeyDefaultLogin]),
43 )
44 if err != nil {
45 return nil, err
46 }
47 if len(creds) <= 0 {
48 return nil, ErrMissingIdentityToken
49 }
50 client := buildClient(creds[0].(*auth.Token))
51 gi.mediator = NewImportMediator(ctx, client, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
52 out := make(chan core.ImportResult)
53 gi.out = out
54
55 go func() {
56 defer close(gi.out)
57 var currBug *cache.BugCache
58 var currEvent ImportEvent
59 var nextEvent ImportEvent
60 var err error
61 for {
62 // We need the current event and one look ahead event.
63 currEvent = nextEvent
64 if currEvent == nil {
65 currEvent = gi.getEventHandleMsgs()
66 }
67 if currEvent == nil {
68 break
69 }
70 nextEvent = gi.getEventHandleMsgs()
71
72 switch event := currEvent.(type) {
73 case MessageEvent:
74 fmt.Println(event.msg)
75 case IssueEvent:
76 // first: commit what is being held in currBug
77 if err = gi.commit(currBug, out); err != nil {
78 out <- core.NewImportError(err, "")
79 return
80 }
81 // second: create new issue
82 switch next := nextEvent.(type) {
83 case IssueEditEvent:
84 // consuming and using next event
85 nextEvent = nil
86 currBug, err = gi.ensureIssue(ctx, repo, &event.issue, &next.userContentEdit)
87 default:
88 currBug, err = gi.ensureIssue(ctx, repo, &event.issue, nil)
89 }
90 if err != nil {
91 err := fmt.Errorf("issue creation: %v", err)
92 out <- core.NewImportError(err, "")
93 return
94 }
95 case IssueEditEvent:
96 err = gi.ensureIssueEdit(ctx, repo, currBug, event.issueId, &event.userContentEdit)
97 if err != nil {
98 err = fmt.Errorf("issue edit: %v", err)
99 out <- core.NewImportError(err, "")
100 return
101 }
102 case TimelineEvent:
103 if next, ok := nextEvent.(CommentEditEvent); ok && event.Typename == "IssueComment" {
104 // consuming and using next event
105 nextEvent = nil
106 err = gi.ensureComment(ctx, repo, currBug, &event.timelineItem.IssueComment, &next.userContentEdit)
107 } else {
108 err = gi.ensureTimelineItem(ctx, repo, currBug, &event.timelineItem)
109 }
110 if err != nil {
111 err = fmt.Errorf("timeline item creation: %v", err)
112 out <- core.NewImportError(err, "")
113 return
114 }
115 case CommentEditEvent:
116 err = gi.ensureCommentEdit(ctx, repo, currBug, event.commentId, &event.userContentEdit)
117 if err != nil {
118 err = fmt.Errorf("comment edit: %v", err)
119 out <- core.NewImportError(err, "")
120 return
121 }
122 default:
123 panic("Unknown event type")
124 }
125 }
126 // commit what is being held in currBug before returning
127 if err = gi.commit(currBug, out); err != nil {
128 out <- core.NewImportError(err, "")
129 }
130 if err = gi.mediator.Error(); err != nil {
131 gi.out <- core.NewImportError(err, "")
132 }
133 }()
134
135 return out, nil
136}
137
138func (gi *githubImporter) getEventHandleMsgs() ImportEvent {
139 for {
140 // read event from import mediator
141 event := gi.mediator.NextImportEvent()
142 // consume (and use) all message events
143 if e, ok := event.(MessageEvent); ok {
144 fmt.Println(e.msg)
145 continue
146 }
147 return event
148 }
149}
150
151func (gi *githubImporter) commit(b *cache.BugCache, out chan<- core.ImportResult) error {
152 if b == nil {
153 return nil
154 }
155 if !b.NeedCommit() {
156 out <- core.NewImportNothing(b.Id(), "no imported operation")
157 return nil
158 } else if err := b.Commit(); err != nil {
159 // commit bug state
160 return fmt.Errorf("bug commit: %v", err)
161 }
162 return nil
163}
164
165func (gi *githubImporter) ensureIssue(ctx context.Context, repo *cache.RepoCache, issue *issue, issueEdit *userContentEdit) (*cache.BugCache, error) {
166 author, err := gi.ensurePerson(ctx, repo, issue.Author)
167 if err != nil {
168 return nil, err
169 }
170
171 // resolve bug
172 b, err := repo.ResolveBugMatcher(func(excerpt *cache.BugExcerpt) bool {
173 return excerpt.CreateMetadata[core.MetaKeyOrigin] == target &&
174 excerpt.CreateMetadata[metaKeyGithubId] == parseId(issue.Id)
175 })
176 if err == nil {
177 return b, nil
178 }
179 if err != bug.ErrBugNotExist {
180 return nil, err
181 }
182
183 // At Github there exist issues with seemingly empty titles. An example is
184 // https://github.com/NixOS/nixpkgs/issues/72730 .
185 // The title provided by the GraphQL API actually consists of a space followed by a
186 // zero width space (U+200B). This title would cause the NewBugRaw() function to
187 // return an error: empty title.
188 title := string(issue.Title)
189 if title == " \u200b" { // U+200B == zero width space
190 title = EMPTY_TITLE_PLACEHOLDER
191 }
192
193 var textInput string
194 if issueEdit != nil {
195 // use the first issue edit: it represents the bug creation itself
196 textInput = string(*issueEdit.Diff)
197 } else {
198 // if there are no issue edits then the issue struct holds the bug creation
199 textInput = string(issue.Body)
200 }
201 cleanText, err := text.Cleanup(textInput)
202 if err != nil {
203 return nil, err
204 }
205
206 // create bug
207 b, _, err = repo.NewBugRaw(
208 author,
209 issue.CreatedAt.Unix(),
210 title, // TODO: this is the *current* title, not the original one
211 cleanText,
212 nil,
213 map[string]string{
214 core.MetaKeyOrigin: target,
215 metaKeyGithubId: parseId(issue.Id),
216 metaKeyGithubUrl: issue.Url.String(),
217 })
218 if err != nil {
219 return nil, err
220 }
221 // importing a new bug
222 gi.out <- core.NewImportBug(b.Id())
223
224 return b, nil
225}
226
227func (gi *githubImporter) ensureIssueEdit(ctx context.Context, repo *cache.RepoCache, bug *cache.BugCache, ghIssueId githubv4.ID, edit *userContentEdit) error {
228 return gi.ensureCommentEdit(ctx, repo, bug, ghIssueId, edit)
229}
230
231func (gi *githubImporter) ensureTimelineItem(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, item *timelineItem) error {
232
233 switch item.Typename {
234 case "IssueComment":
235 err := gi.ensureComment(ctx, repo, b, &item.IssueComment, nil)
236 if err != nil {
237 return fmt.Errorf("timeline comment creation: %v", err)
238 }
239 return nil
240
241 case "LabeledEvent":
242 id := parseId(item.LabeledEvent.Id)
243 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
244 if err == nil {
245 return nil
246 }
247
248 if err != cache.ErrNoMatchingOp {
249 return err
250 }
251 author, err := gi.ensurePerson(ctx, repo, item.LabeledEvent.Actor)
252 if err != nil {
253 return err
254 }
255 op, err := b.ForceChangeLabelsRaw(
256 author,
257 item.LabeledEvent.CreatedAt.Unix(),
258 []string{
259 string(item.LabeledEvent.Label.Name),
260 },
261 nil,
262 map[string]string{metaKeyGithubId: id},
263 )
264 if err != nil {
265 return err
266 }
267
268 gi.out <- core.NewImportLabelChange(op.Id())
269 return nil
270
271 case "UnlabeledEvent":
272 id := parseId(item.UnlabeledEvent.Id)
273 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
274 if err == nil {
275 return nil
276 }
277 if err != cache.ErrNoMatchingOp {
278 return err
279 }
280 author, err := gi.ensurePerson(ctx, repo, item.UnlabeledEvent.Actor)
281 if err != nil {
282 return err
283 }
284
285 op, err := b.ForceChangeLabelsRaw(
286 author,
287 item.UnlabeledEvent.CreatedAt.Unix(),
288 nil,
289 []string{
290 string(item.UnlabeledEvent.Label.Name),
291 },
292 map[string]string{metaKeyGithubId: id},
293 )
294 if err != nil {
295 return err
296 }
297
298 gi.out <- core.NewImportLabelChange(op.Id())
299 return nil
300
301 case "ClosedEvent":
302 id := parseId(item.ClosedEvent.Id)
303 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
304 if err != cache.ErrNoMatchingOp {
305 return err
306 }
307 if err == nil {
308 return nil
309 }
310 author, err := gi.ensurePerson(ctx, repo, item.ClosedEvent.Actor)
311 if err != nil {
312 return err
313 }
314 op, err := b.CloseRaw(
315 author,
316 item.ClosedEvent.CreatedAt.Unix(),
317 map[string]string{metaKeyGithubId: id},
318 )
319
320 if err != nil {
321 return err
322 }
323
324 gi.out <- core.NewImportStatusChange(op.Id())
325 return nil
326
327 case "ReopenedEvent":
328 id := parseId(item.ReopenedEvent.Id)
329 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
330 if err != cache.ErrNoMatchingOp {
331 return err
332 }
333 if err == nil {
334 return nil
335 }
336 author, err := gi.ensurePerson(ctx, repo, item.ReopenedEvent.Actor)
337 if err != nil {
338 return err
339 }
340 op, err := b.OpenRaw(
341 author,
342 item.ReopenedEvent.CreatedAt.Unix(),
343 map[string]string{metaKeyGithubId: id},
344 )
345
346 if err != nil {
347 return err
348 }
349
350 gi.out <- core.NewImportStatusChange(op.Id())
351 return nil
352
353 case "RenamedTitleEvent":
354 id := parseId(item.RenamedTitleEvent.Id)
355 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
356 if err != cache.ErrNoMatchingOp {
357 return err
358 }
359 if err == nil {
360 return nil
361 }
362 author, err := gi.ensurePerson(ctx, repo, item.RenamedTitleEvent.Actor)
363 if err != nil {
364 return err
365 }
366
367 // At Github there exist issues with seemingly empty titles. An example is
368 // https://github.com/NixOS/nixpkgs/issues/72730 .
369 // The title provided by the GraphQL API actually consists of a space followed
370 // by a zero width space (U+200B). This title would cause the NewBugRaw()
371 // function to return an error: empty title.
372 title := string(item.RenamedTitleEvent.CurrentTitle)
373 if title == " \u200b" { // U+200B == zero width space
374 title = EMPTY_TITLE_PLACEHOLDER
375 }
376
377 op, err := b.SetTitleRaw(
378 author,
379 item.RenamedTitleEvent.CreatedAt.Unix(),
380 title,
381 map[string]string{metaKeyGithubId: id},
382 )
383 if err != nil {
384 return err
385 }
386
387 gi.out <- core.NewImportTitleEdition(op.Id())
388 return nil
389 }
390
391 return nil
392}
393
394func (gi *githubImporter) ensureCommentEdit(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, ghTargetId githubv4.ID, edit *userContentEdit) error {
395 // find comment
396 target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(ghTargetId))
397 if err != nil {
398 return err
399 }
400 _, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
401 if err == nil {
402 return nil
403 }
404 if err != cache.ErrNoMatchingOp {
405 // real error
406 return err
407 }
408
409 editor, err := gi.ensurePerson(ctx, repo, edit.Editor)
410 if err != nil {
411 return err
412 }
413
414 if edit.DeletedAt != nil {
415 // comment deletion, not supported yet
416 return nil
417 }
418
419 cleanText, err := text.Cleanup(string(*edit.Diff))
420 if err != nil {
421 return err
422 }
423
424 // comment edition
425 op, err := b.EditCommentRaw(
426 editor,
427 edit.CreatedAt.Unix(),
428 target,
429 cleanText,
430 map[string]string{
431 metaKeyGithubId: parseId(edit.Id),
432 },
433 )
434
435 if err != nil {
436 return err
437 }
438
439 gi.out <- core.NewImportCommentEdition(op.Id())
440 return nil
441}
442
443func (gi *githubImporter) ensureComment(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, comment *issueComment, firstEdit *userContentEdit) error {
444 author, err := gi.ensurePerson(ctx, repo, comment.Author)
445 if err != nil {
446 return err
447 }
448
449 _, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(comment.Id))
450 if err == nil {
451 return nil
452 }
453 if err != cache.ErrNoMatchingOp {
454 // real error
455 return err
456 }
457
458 var textInput string
459 if firstEdit != nil {
460 // use the first comment edit: it represents the comment creation itself
461 textInput = string(*firstEdit.Diff)
462 } else {
463 // if there are not comment edits, then the comment struct holds the comment creation
464 textInput = string(comment.Body)
465 }
466 cleanText, err := text.Cleanup(textInput)
467 if err != nil {
468 return err
469 }
470
471 // add comment operation
472 op, err := b.AddCommentRaw(
473 author,
474 comment.CreatedAt.Unix(),
475 cleanText,
476 nil,
477 map[string]string{
478 metaKeyGithubId: parseId(comment.Id),
479 metaKeyGithubUrl: comment.Url.String(),
480 },
481 )
482 if err != nil {
483 return err
484 }
485
486 gi.out <- core.NewImportComment(op.Id())
487 return nil
488}
489
490// ensurePerson create a bug.Person from the Github data
491func (gi *githubImporter) ensurePerson(ctx context.Context, repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
492 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
493 // in it's UI. So we need a special case to get it.
494 if actor == nil {
495 return gi.getGhost(ctx, repo)
496 }
497
498 // Look first in the cache
499 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
500 if err == nil {
501 return i, nil
502 }
503 if entity.IsErrMultipleMatch(err) {
504 return nil, err
505 }
506
507 // importing a new identity
508 var name string
509 var email string
510
511 switch actor.Typename {
512 case "User":
513 if actor.User.Name != nil {
514 name = string(*(actor.User.Name))
515 }
516 email = string(actor.User.Email)
517 case "Organization":
518 if actor.Organization.Name != nil {
519 name = string(*(actor.Organization.Name))
520 }
521 if actor.Organization.Email != nil {
522 email = string(*(actor.Organization.Email))
523 }
524 case "Bot":
525 }
526
527 // Name is not necessarily set, fallback to login as a name is required in the identity
528 if name == "" {
529 name = string(actor.Login)
530 }
531
532 i, err = repo.NewIdentityRaw(
533 name,
534 email,
535 string(actor.Login),
536 string(actor.AvatarUrl),
537 map[string]string{
538 metaKeyGithubLogin: string(actor.Login),
539 },
540 )
541
542 if err != nil {
543 return nil, err
544 }
545
546 gi.out <- core.NewImportIdentity(i.Id())
547 return i, nil
548}
549
550func (gi *githubImporter) getGhost(ctx context.Context, repo *cache.RepoCache) (*cache.IdentityCache, error) {
551 loginName := "ghost"
552 // Look first in the cache
553 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, loginName)
554 if err == nil {
555 return i, nil
556 }
557 if entity.IsErrMultipleMatch(err) {
558 return nil, err
559 }
560 user, err := gi.mediator.User(ctx, loginName)
561 userName := ""
562 if user.Name != nil {
563 userName = string(*user.Name)
564 }
565 return repo.NewIdentityRaw(
566 userName,
567 "",
568 string(user.Login),
569 string(user.AvatarUrl),
570 map[string]string{
571 metaKeyGithubLogin: string(user.Login),
572 },
573 )
574}
575
576// parseId converts the unusable githubv4.ID (an interface{}) into a string
577func parseId(id githubv4.ID) string {
578 return fmt.Sprintf("%v", id)
579}