1package github
2
3import (
4 "context"
5 "fmt"
6 "time"
7
8 "github.com/shurcooL/githubv4"
9
10 "github.com/MichaelMure/git-bug/bridge/core"
11 "github.com/MichaelMure/git-bug/bridge/core/auth"
12 "github.com/MichaelMure/git-bug/bug"
13 "github.com/MichaelMure/git-bug/cache"
14 "github.com/MichaelMure/git-bug/entity"
15 "github.com/MichaelMure/git-bug/util/text"
16)
17
18const EMPTY_TITLE_PLACEHOLDER = "<empty string>"
19
20// githubImporter implement the Importer interface
21type githubImporter struct {
22 conf core.Configuration
23
24 // mediator to access the Github API
25 mediator *importMediator
26
27 // send only channel
28 out chan<- core.ImportResult
29}
30
31func (gi *githubImporter) Init(_ context.Context, _ *cache.RepoCache, conf core.Configuration) error {
32 gi.conf = conf
33 return nil
34}
35
36// ImportAll iterate over all the configured repository issues and ensure the creation of the
37// missing issues / timeline items / edits / label events ...
38func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
39 creds, err := auth.List(repo,
40 auth.WithTarget(target),
41 auth.WithKind(auth.KindToken),
42 auth.WithMeta(auth.MetaKeyLogin, gi.conf[confKeyDefaultLogin]),
43 )
44 if err != nil {
45 return nil, err
46 }
47 if len(creds) <= 0 {
48 return nil, ErrMissingIdentityToken
49 }
50 client := buildClient(creds[0].(*auth.Token))
51 gi.mediator = NewImportMediator(ctx, client, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
52 out := make(chan core.ImportResult)
53 gi.out = out
54
55 go func() {
56 defer close(gi.out)
57
58 // Loop over all matching issues
59 for bundle := range gi.mediator.Issues() {
60 issue := bundle.issue
61 issueEdits := bundle.issueEdits
62 timelineBundles := bundle.timelineBundles
63 // create issue
64 b, err := gi.ensureIssue(ctx, repo, &issue, issueEdits)
65 if err != nil {
66 err := fmt.Errorf("issue creation: %v", err)
67 out <- core.NewImportError(err, "")
68 return
69 }
70
71 // loop over timeline items
72 for bundle := range timelineBundles {
73 item := bundle.timelineItem
74 edits := bundle.userContentEdits
75 err := gi.ensureTimelineItem(ctx, repo, b, &item, edits)
76 if err != nil {
77 err = fmt.Errorf("timeline item creation: %v", err)
78 out <- core.NewImportError(err, "")
79 return
80 }
81 }
82
83 if !b.NeedCommit() {
84 out <- core.NewImportNothing(b.Id(), "no imported operation")
85 } else if err := b.Commit(); err != nil {
86 // commit bug state
87 err = fmt.Errorf("bug commit: %v", err)
88 out <- core.NewImportError(err, "")
89 return
90 }
91 }
92
93 if err := gi.mediator.Error(); err != nil {
94 gi.out <- core.NewImportError(err, "")
95 }
96 }()
97
98 return out, nil
99}
100
101func (gi *githubImporter) ensureIssue(ctx context.Context, repo *cache.RepoCache, issue *issue, issueEdits <-chan userContentEdit) (*cache.BugCache, error) {
102 author, err := gi.ensurePerson(ctx, repo, issue.Author)
103 if err != nil {
104 return nil, err
105 }
106
107 // resolve bug
108 b, err := repo.ResolveBugMatcher(func(excerpt *cache.BugExcerpt) bool {
109 return excerpt.CreateMetadata[core.MetaKeyOrigin] == target &&
110 excerpt.CreateMetadata[metaKeyGithubId] == parseId(issue.Id)
111 })
112 if err != nil && err != bug.ErrBugNotExist {
113 return nil, err
114 }
115
116 // get first issue edit
117 // if it exists, then it holds the bug creation
118 firstEdit, hasEdit := <-issueEdits
119
120 // At Github there exist issues with seemingly empty titles. An example is
121 // https://github.com/NixOS/nixpkgs/issues/72730 .
122 // The title provided by the GraphQL API actually consists of a space followed by a
123 // zero width space (U+200B). This title would cause the NewBugRaw() function to
124 // return an error: empty title.
125 title := string(issue.Title)
126 if title == " \u200b" { // U+200B == zero width space
127 title = EMPTY_TITLE_PLACEHOLDER
128 }
129
130 if err == bug.ErrBugNotExist {
131 var textInput string
132 if hasEdit {
133 // use the first issue edit: it represents the bug creation itself
134 textInput = string(*firstEdit.Diff)
135 } else {
136 // if there are no issue edits then the issue struct holds the bug creation
137 textInput = string(issue.Body)
138 }
139 cleanText, err := text.Cleanup(textInput)
140 if err != nil {
141 return nil, err
142 }
143 // create bug
144 b, _, err = repo.NewBugRaw(
145 author,
146 issue.CreatedAt.Unix(),
147 title, // TODO: this is the *current* title, not the original one
148 cleanText,
149 nil,
150 map[string]string{
151 core.MetaKeyOrigin: target,
152 metaKeyGithubId: parseId(issue.Id),
153 metaKeyGithubUrl: issue.Url.String(),
154 })
155 if err != nil {
156 return nil, err
157 }
158 // importing a new bug
159 gi.out <- core.NewImportBug(b.Id())
160 }
161 if b == nil {
162 return nil, fmt.Errorf("finding or creating issue")
163 }
164 // process remaining issue edits, if they exist
165 for edit := range issueEdits {
166 // other edits will be added as CommentEdit operations
167 target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(issue.Id))
168 if err == cache.ErrNoMatchingOp {
169 // original comment is missing somehow, issuing a warning
170 gi.out <- core.NewImportWarning(fmt.Errorf("comment ID %s to edit is missing", parseId(issue.Id)), b.Id())
171 continue
172 }
173 if err != nil {
174 return nil, err
175 }
176
177 err = gi.ensureCommentEdit(ctx, repo, b, target, &edit)
178 if err != nil {
179 return nil, err
180 }
181 }
182 return b, nil
183}
184
185func (gi *githubImporter) ensureTimelineItem(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, item *timelineItem, commentEdits <-chan userContentEdit) error {
186
187 switch item.Typename {
188 case "IssueComment":
189 err := gi.ensureComment(ctx, repo, b, &item.IssueComment, commentEdits)
190 if err != nil {
191 return fmt.Errorf("timeline comment creation: %v", err)
192 }
193 return nil
194
195 case "LabeledEvent":
196 id := parseId(item.LabeledEvent.Id)
197 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
198 if err == nil {
199 return nil
200 }
201
202 if err != cache.ErrNoMatchingOp {
203 return err
204 }
205 author, err := gi.ensurePerson(ctx, repo, item.LabeledEvent.Actor)
206 if err != nil {
207 return err
208 }
209 op, err := b.ForceChangeLabelsRaw(
210 author,
211 item.LabeledEvent.CreatedAt.Unix(),
212 []string{
213 string(item.LabeledEvent.Label.Name),
214 },
215 nil,
216 map[string]string{metaKeyGithubId: id},
217 )
218 if err != nil {
219 return err
220 }
221
222 gi.out <- core.NewImportLabelChange(op.Id())
223 return nil
224
225 case "UnlabeledEvent":
226 id := parseId(item.UnlabeledEvent.Id)
227 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
228 if err == nil {
229 return nil
230 }
231 if err != cache.ErrNoMatchingOp {
232 return err
233 }
234 author, err := gi.ensurePerson(ctx, repo, item.UnlabeledEvent.Actor)
235 if err != nil {
236 return err
237 }
238
239 op, err := b.ForceChangeLabelsRaw(
240 author,
241 item.UnlabeledEvent.CreatedAt.Unix(),
242 nil,
243 []string{
244 string(item.UnlabeledEvent.Label.Name),
245 },
246 map[string]string{metaKeyGithubId: id},
247 )
248 if err != nil {
249 return err
250 }
251
252 gi.out <- core.NewImportLabelChange(op.Id())
253 return nil
254
255 case "ClosedEvent":
256 id := parseId(item.ClosedEvent.Id)
257 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
258 if err != cache.ErrNoMatchingOp {
259 return err
260 }
261 if err == nil {
262 return nil
263 }
264 author, err := gi.ensurePerson(ctx, repo, item.ClosedEvent.Actor)
265 if err != nil {
266 return err
267 }
268 op, err := b.CloseRaw(
269 author,
270 item.ClosedEvent.CreatedAt.Unix(),
271 map[string]string{metaKeyGithubId: id},
272 )
273
274 if err != nil {
275 return err
276 }
277
278 gi.out <- core.NewImportStatusChange(op.Id())
279 return nil
280
281 case "ReopenedEvent":
282 id := parseId(item.ReopenedEvent.Id)
283 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
284 if err != cache.ErrNoMatchingOp {
285 return err
286 }
287 if err == nil {
288 return nil
289 }
290 author, err := gi.ensurePerson(ctx, repo, item.ReopenedEvent.Actor)
291 if err != nil {
292 return err
293 }
294 op, err := b.OpenRaw(
295 author,
296 item.ReopenedEvent.CreatedAt.Unix(),
297 map[string]string{metaKeyGithubId: id},
298 )
299
300 if err != nil {
301 return err
302 }
303
304 gi.out <- core.NewImportStatusChange(op.Id())
305 return nil
306
307 case "RenamedTitleEvent":
308 id := parseId(item.RenamedTitleEvent.Id)
309 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
310 if err != cache.ErrNoMatchingOp {
311 return err
312 }
313 if err == nil {
314 return nil
315 }
316 author, err := gi.ensurePerson(ctx, repo, item.RenamedTitleEvent.Actor)
317 if err != nil {
318 return err
319 }
320
321 // At Github there exist issues with seemingly empty titles. An example is
322 // https://github.com/NixOS/nixpkgs/issues/72730 .
323 // The title provided by the GraphQL API actually consists of a space followed
324 // by a zero width space (U+200B). This title would cause the NewBugRaw()
325 // function to return an error: empty title.
326 title := string(item.RenamedTitleEvent.CurrentTitle)
327 if title == " \u200b" { // U+200B == zero width space
328 title = EMPTY_TITLE_PLACEHOLDER
329 }
330
331 op, err := b.SetTitleRaw(
332 author,
333 item.RenamedTitleEvent.CreatedAt.Unix(),
334 title,
335 map[string]string{metaKeyGithubId: id},
336 )
337 if err != nil {
338 return err
339 }
340
341 gi.out <- core.NewImportTitleEdition(op.Id())
342 return nil
343 }
344
345 return nil
346}
347
348func (gi *githubImporter) ensureComment(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, comment *issueComment, commentEdits <-chan userContentEdit) error {
349 author, err := gi.ensurePerson(ctx, repo, comment.Author)
350 if err != nil {
351 return err
352 }
353
354 targetOpID, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(comment.Id))
355 if err != nil && err != cache.ErrNoMatchingOp {
356 // real error
357 return err
358 }
359 firstEdit, hasEdit := <-commentEdits
360 if err == cache.ErrNoMatchingOp {
361 var textInput string
362 if hasEdit {
363 // use the first comment edit: it represents the comment creation itself
364 textInput = string(*firstEdit.Diff)
365 } else {
366 // if there are not comment edits, then the comment struct holds the comment creation
367 textInput = string(comment.Body)
368 }
369 cleanText, err := text.Cleanup(textInput)
370 if err != nil {
371 return err
372 }
373
374 // add comment operation
375 op, err := b.AddCommentRaw(
376 author,
377 comment.CreatedAt.Unix(),
378 cleanText,
379 nil,
380 map[string]string{
381 metaKeyGithubId: parseId(comment.Id),
382 metaKeyGithubUrl: comment.Url.String(),
383 },
384 )
385 if err != nil {
386 return err
387 }
388
389 gi.out <- core.NewImportComment(op.Id())
390 targetOpID = op.Id()
391 }
392 if targetOpID == "" {
393 return fmt.Errorf("finding or creating issue comment")
394 }
395 // process remaining comment edits, if they exist
396 for edit := range commentEdits {
397 // ensure editor identity
398 _, err := gi.ensurePerson(ctx, repo, edit.Editor)
399 if err != nil {
400 return err
401 }
402
403 err = gi.ensureCommentEdit(ctx, repo, b, targetOpID, &edit)
404 if err != nil {
405 return err
406 }
407 }
408 return nil
409}
410
411func (gi *githubImporter) ensureCommentEdit(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, target entity.Id, edit *userContentEdit) error {
412 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
413 if err == nil {
414 return nil
415 }
416 if err != cache.ErrNoMatchingOp {
417 // real error
418 return err
419 }
420
421 editor, err := gi.ensurePerson(ctx, repo, edit.Editor)
422 if err != nil {
423 return err
424 }
425
426 switch {
427 case edit.DeletedAt != nil:
428 // comment deletion, not supported yet
429 return nil
430
431 case edit.DeletedAt == nil:
432
433 cleanText, err := text.Cleanup(string(*edit.Diff))
434 if err != nil {
435 return err
436 }
437
438 // comment edition
439 op, err := b.EditCommentRaw(
440 editor,
441 edit.CreatedAt.Unix(),
442 target,
443 cleanText,
444 map[string]string{
445 metaKeyGithubId: parseId(edit.Id),
446 },
447 )
448
449 if err != nil {
450 return err
451 }
452
453 gi.out <- core.NewImportCommentEdition(op.Id())
454 return nil
455 }
456 return nil
457}
458
459// ensurePerson create a bug.Person from the Github data
460func (gi *githubImporter) ensurePerson(ctx context.Context, repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
461 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
462 // in it's UI. So we need a special case to get it.
463 if actor == nil {
464 return gi.getGhost(ctx, repo)
465 }
466
467 // Look first in the cache
468 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
469 if err == nil {
470 return i, nil
471 }
472 if entity.IsErrMultipleMatch(err) {
473 return nil, err
474 }
475
476 // importing a new identity
477 var name string
478 var email string
479
480 switch actor.Typename {
481 case "User":
482 if actor.User.Name != nil {
483 name = string(*(actor.User.Name))
484 }
485 email = string(actor.User.Email)
486 case "Organization":
487 if actor.Organization.Name != nil {
488 name = string(*(actor.Organization.Name))
489 }
490 if actor.Organization.Email != nil {
491 email = string(*(actor.Organization.Email))
492 }
493 case "Bot":
494 }
495
496 // Name is not necessarily set, fallback to login as a name is required in the identity
497 if name == "" {
498 name = string(actor.Login)
499 }
500
501 i, err = repo.NewIdentityRaw(
502 name,
503 email,
504 string(actor.Login),
505 string(actor.AvatarUrl),
506 map[string]string{
507 metaKeyGithubLogin: string(actor.Login),
508 },
509 )
510
511 if err != nil {
512 return nil, err
513 }
514
515 gi.out <- core.NewImportIdentity(i.Id())
516 return i, nil
517}
518
519func (gi *githubImporter) getGhost(ctx context.Context, repo *cache.RepoCache) (*cache.IdentityCache, error) {
520 loginName := "ghost"
521 // Look first in the cache
522 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, loginName)
523 if err == nil {
524 return i, nil
525 }
526 if entity.IsErrMultipleMatch(err) {
527 return nil, err
528 }
529 user, err := gi.mediator.User(ctx, loginName)
530 userName := ""
531 if user.Name != nil {
532 userName = string(*user.Name)
533 }
534 return repo.NewIdentityRaw(
535 userName,
536 "",
537 string(user.Login),
538 string(user.AvatarUrl),
539 map[string]string{
540 metaKeyGithubLogin: string(user.Login),
541 },
542 )
543}
544
545// parseId converts the unusable githubv4.ID (an interface{}) into a string
546func parseId(id githubv4.ID) string {
547 return fmt.Sprintf("%v", id)
548}