1package github
2
3import (
4 "context"
5 "fmt"
6 "time"
7
8 "github.com/shurcooL/githubv4"
9
10 "github.com/MichaelMure/git-bug/bridge/core"
11 "github.com/MichaelMure/git-bug/bug"
12 "github.com/MichaelMure/git-bug/cache"
13 "github.com/MichaelMure/git-bug/entity"
14 "github.com/MichaelMure/git-bug/util/text"
15)
16
17const (
18 metaKeyGithubId = "github-id"
19 metaKeyGithubUrl = "github-url"
20 metaKeyGithubLogin = "github-login"
21)
22
23// githubImporter implement the Importer interface
24type githubImporter struct {
25 conf core.Configuration
26
27 // iterator
28 iterator *iterator
29
30 // send only channel
31 out chan<- core.ImportResult
32}
33
34func (gi *githubImporter) Init(conf core.Configuration) error {
35 gi.conf = conf
36 return nil
37}
38
39// ImportAll iterate over all the configured repository issues and ensure the creation of the
40// missing issues / timeline items / edits / label events ...
41func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
42 gi.iterator = NewIterator(ctx, 10, gi.conf[keyOwner], gi.conf[keyProject], gi.conf[core.ConfigKeyToken], since)
43 out := make(chan core.ImportResult)
44 gi.out = out
45
46 go func() {
47 defer close(gi.out)
48
49 // Loop over all matching issues
50 for gi.iterator.NextIssue() {
51 issue := gi.iterator.IssueValue()
52 // create issue
53 b, err := gi.ensureIssue(repo, issue)
54 if err != nil {
55 err := fmt.Errorf("issue creation: %v", err)
56 out <- core.NewImportError(err, "")
57 return
58 }
59
60 // loop over timeline items
61 for gi.iterator.NextTimelineItem() {
62 item := gi.iterator.TimelineItemValue()
63 err := gi.ensureTimelineItem(repo, b, item)
64 if err != nil {
65 err = fmt.Errorf("timeline item creation: %v", err)
66 out <- core.NewImportError(err, "")
67 return
68 }
69 }
70
71 if !b.NeedCommit() {
72 out <- core.NewImportNothing(b.Id(), "no imported operation")
73 } else if err := b.Commit(); err != nil {
74 // commit bug state
75 err = fmt.Errorf("bug commit: %v", err)
76 out <- core.NewImportError(err, "")
77 return
78 }
79 }
80
81 if err := gi.iterator.Error(); err != nil {
82 gi.out <- core.NewImportError(err, "")
83 }
84 }()
85
86 return out, nil
87}
88
89func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline) (*cache.BugCache, error) {
90 // ensure issue author
91 author, err := gi.ensurePerson(repo, issue.Author)
92 if err != nil {
93 return nil, err
94 }
95
96 // resolve bug
97 b, err := repo.ResolveBugCreateMetadata(metaKeyGithubUrl, issue.Url.String())
98 if err != nil && err != bug.ErrBugNotExist {
99 return nil, err
100 }
101
102 // get issue edits
103 var issueEdits []userContentEdit
104 for gi.iterator.NextIssueEdit() {
105 issueEdits = append(issueEdits, gi.iterator.IssueEditValue())
106 }
107
108 // if issueEdits is empty
109 if len(issueEdits) == 0 {
110 if err == bug.ErrBugNotExist {
111 cleanText, err := text.Cleanup(string(issue.Body))
112 if err != nil {
113 return nil, err
114 }
115
116 // create bug
117 b, _, err = repo.NewBugRaw(
118 author,
119 issue.CreatedAt.Unix(),
120 issue.Title,
121 cleanText,
122 nil,
123 map[string]string{
124 core.MetaKeyOrigin: target,
125 metaKeyGithubId: parseId(issue.Id),
126 metaKeyGithubUrl: issue.Url.String(),
127 })
128 if err != nil {
129 return nil, err
130 }
131
132 // importing a new bug
133 gi.out <- core.NewImportBug(b.Id())
134 }
135 } else {
136 // create bug from given issueEdits
137 for i, edit := range issueEdits {
138 if i == 0 && b != nil {
139 // The first edit in the github result is the issue creation itself, we already have that
140 continue
141 }
142
143 cleanText, err := text.Cleanup(string(*edit.Diff))
144 if err != nil {
145 return nil, err
146 }
147
148 // if the bug doesn't exist
149 if b == nil {
150 // we create the bug as soon as we have a legit first edition
151 b, _, err = repo.NewBugRaw(
152 author,
153 issue.CreatedAt.Unix(),
154 issue.Title,
155 cleanText,
156 nil,
157 map[string]string{
158 core.MetaKeyOrigin: target,
159 metaKeyGithubId: parseId(issue.Id),
160 metaKeyGithubUrl: issue.Url.String(),
161 },
162 )
163
164 if err != nil {
165 return nil, err
166 }
167 // importing a new bug
168 gi.out <- core.NewImportBug(b.Id())
169 continue
170 }
171
172 // other edits will be added as CommentEdit operations
173 target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(issue.Id))
174 if err != nil {
175 return nil, err
176 }
177
178 err = gi.ensureCommentEdit(repo, b, target, edit)
179 if err != nil {
180 return nil, err
181 }
182 }
183 }
184
185 return b, nil
186}
187
188func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, item timelineItem) error {
189
190 switch item.Typename {
191 case "IssueComment":
192 // collect all comment edits
193 var commentEdits []userContentEdit
194 for gi.iterator.NextCommentEdit() {
195 commentEdits = append(commentEdits, gi.iterator.CommentEditValue())
196 }
197
198 // ensureTimelineComment send import events over out chanel
199 err := gi.ensureTimelineComment(repo, b, item.IssueComment, commentEdits)
200 if err != nil {
201 return fmt.Errorf("timeline comment creation: %v", err)
202 }
203 return nil
204
205 case "LabeledEvent":
206 id := parseId(item.LabeledEvent.Id)
207 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
208 if err == nil {
209 return nil
210 }
211
212 if err != cache.ErrNoMatchingOp {
213 return err
214 }
215 author, err := gi.ensurePerson(repo, item.LabeledEvent.Actor)
216 if err != nil {
217 return err
218 }
219 op, err := b.ForceChangeLabelsRaw(
220 author,
221 item.LabeledEvent.CreatedAt.Unix(),
222 []string{
223 string(item.LabeledEvent.Label.Name),
224 },
225 nil,
226 map[string]string{metaKeyGithubId: id},
227 )
228 if err != nil {
229 return err
230 }
231
232 gi.out <- core.NewImportLabelChange(op.Id())
233 return nil
234
235 case "UnlabeledEvent":
236 id := parseId(item.UnlabeledEvent.Id)
237 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
238 if err == nil {
239 return nil
240 }
241 if err != cache.ErrNoMatchingOp {
242 return err
243 }
244 author, err := gi.ensurePerson(repo, item.UnlabeledEvent.Actor)
245 if err != nil {
246 return err
247 }
248
249 op, err := b.ForceChangeLabelsRaw(
250 author,
251 item.UnlabeledEvent.CreatedAt.Unix(),
252 nil,
253 []string{
254 string(item.UnlabeledEvent.Label.Name),
255 },
256 map[string]string{metaKeyGithubId: id},
257 )
258 if err != nil {
259 return err
260 }
261
262 gi.out <- core.NewImportLabelChange(op.Id())
263 return nil
264
265 case "ClosedEvent":
266 id := parseId(item.ClosedEvent.Id)
267 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
268 if err != cache.ErrNoMatchingOp {
269 return err
270 }
271 if err == nil {
272 return nil
273 }
274 author, err := gi.ensurePerson(repo, item.ClosedEvent.Actor)
275 if err != nil {
276 return err
277 }
278 op, err := b.CloseRaw(
279 author,
280 item.ClosedEvent.CreatedAt.Unix(),
281 map[string]string{metaKeyGithubId: id},
282 )
283
284 if err != nil {
285 return err
286 }
287
288 gi.out <- core.NewImportStatusChange(op.Id())
289 return nil
290
291 case "ReopenedEvent":
292 id := parseId(item.ReopenedEvent.Id)
293 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
294 if err != cache.ErrNoMatchingOp {
295 return err
296 }
297 if err == nil {
298 return nil
299 }
300 author, err := gi.ensurePerson(repo, item.ReopenedEvent.Actor)
301 if err != nil {
302 return err
303 }
304 op, err := b.OpenRaw(
305 author,
306 item.ReopenedEvent.CreatedAt.Unix(),
307 map[string]string{metaKeyGithubId: id},
308 )
309
310 if err != nil {
311 return err
312 }
313
314 gi.out <- core.NewImportStatusChange(op.Id())
315 return nil
316
317 case "RenamedTitleEvent":
318 id := parseId(item.RenamedTitleEvent.Id)
319 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
320 if err != cache.ErrNoMatchingOp {
321 return err
322 }
323 if err == nil {
324 return nil
325 }
326 author, err := gi.ensurePerson(repo, item.RenamedTitleEvent.Actor)
327 if err != nil {
328 return err
329 }
330 op, err := b.SetTitleRaw(
331 author,
332 item.RenamedTitleEvent.CreatedAt.Unix(),
333 string(item.RenamedTitleEvent.CurrentTitle),
334 map[string]string{metaKeyGithubId: id},
335 )
336 if err != nil {
337 return err
338 }
339
340 gi.out <- core.NewImportTitleEdition(op.Id())
341 return nil
342 }
343
344 return nil
345}
346
347func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache.BugCache, item issueComment, edits []userContentEdit) error {
348 // ensure person
349 author, err := gi.ensurePerson(repo, item.Author)
350 if err != nil {
351 return err
352 }
353
354 targetOpID, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(item.Id))
355 if err != nil && err != cache.ErrNoMatchingOp {
356 // real error
357 return err
358 }
359
360 // if no edits are given we create the comment
361 if len(edits) == 0 {
362 if err == cache.ErrNoMatchingOp {
363 cleanText, err := text.Cleanup(string(item.Body))
364 if err != nil {
365 return err
366 }
367
368 // add comment operation
369 op, err := b.AddCommentRaw(
370 author,
371 item.CreatedAt.Unix(),
372 cleanText,
373 nil,
374 map[string]string{
375 metaKeyGithubId: parseId(item.Id),
376 metaKeyGithubUrl: parseId(item.Url.String()),
377 },
378 )
379 if err != nil {
380 return err
381 }
382
383 gi.out <- core.NewImportComment(op.Id())
384 return nil
385 }
386
387 } else {
388 for i, edit := range edits {
389 if i == 0 && targetOpID != "" {
390 // The first edit in the github result is the comment creation itself, we already have that
391 continue
392 }
393
394 // ensure editor identity
395 editor, err := gi.ensurePerson(repo, edit.Editor)
396 if err != nil {
397 return err
398 }
399
400 // create comment when target is empty
401 if targetOpID == "" {
402 cleanText, err := text.Cleanup(string(*edit.Diff))
403 if err != nil {
404 return err
405 }
406
407 op, err := b.AddCommentRaw(
408 editor,
409 edit.CreatedAt.Unix(),
410 cleanText,
411 nil,
412 map[string]string{
413 metaKeyGithubId: parseId(item.Id),
414 metaKeyGithubUrl: item.Url.String(),
415 },
416 )
417 if err != nil {
418 return err
419 }
420 gi.out <- core.NewImportComment(op.Id())
421
422 // set target for the nexr edit now that the comment is created
423 targetOpID = op.Id()
424 continue
425 }
426
427 err = gi.ensureCommentEdit(repo, b, targetOpID, edit)
428 if err != nil {
429 return err
430 }
431 }
432 }
433 return nil
434}
435
436func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugCache, target entity.Id, edit userContentEdit) error {
437 _, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
438 if err == nil {
439 return nil
440 }
441 if err != cache.ErrNoMatchingOp {
442 // real error
443 return err
444 }
445
446 editor, err := gi.ensurePerson(repo, edit.Editor)
447 if err != nil {
448 return err
449 }
450
451 switch {
452 case edit.DeletedAt != nil:
453 // comment deletion, not supported yet
454 return nil
455
456 case edit.DeletedAt == nil:
457
458 cleanText, err := text.Cleanup(string(*edit.Diff))
459 if err != nil {
460 return err
461 }
462
463 // comment edition
464 op, err := b.EditCommentRaw(
465 editor,
466 edit.CreatedAt.Unix(),
467 target,
468 cleanText,
469 map[string]string{
470 metaKeyGithubId: parseId(edit.Id),
471 },
472 )
473
474 if err != nil {
475 return err
476 }
477
478 gi.out <- core.NewImportCommentEdition(op.Id())
479 return nil
480 }
481 return nil
482}
483
484// ensurePerson create a bug.Person from the Github data
485func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
486 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
487 // in it's UI. So we need a special case to get it.
488 if actor == nil {
489 return gi.getGhost(repo)
490 }
491
492 // Look first in the cache
493 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
494 if err == nil {
495 return i, nil
496 }
497 if _, ok := err.(entity.ErrMultipleMatch); ok {
498 return nil, err
499 }
500
501 // importing a new identity
502
503 var name string
504 var email string
505
506 switch actor.Typename {
507 case "User":
508 if actor.User.Name != nil {
509 name = string(*(actor.User.Name))
510 }
511 email = string(actor.User.Email)
512 case "Organization":
513 if actor.Organization.Name != nil {
514 name = string(*(actor.Organization.Name))
515 }
516 if actor.Organization.Email != nil {
517 email = string(*(actor.Organization.Email))
518 }
519 case "Bot":
520 }
521
522 i, err = repo.NewIdentityRaw(
523 name,
524 email,
525 string(actor.Login),
526 string(actor.AvatarUrl),
527 map[string]string{
528 metaKeyGithubLogin: string(actor.Login),
529 },
530 )
531
532 if err != nil {
533 return nil, err
534 }
535
536 gi.out <- core.NewImportIdentity(i.Id())
537 return i, nil
538}
539
540func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, error) {
541 // Look first in the cache
542 i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, "ghost")
543 if err == nil {
544 return i, nil
545 }
546 if _, ok := err.(entity.ErrMultipleMatch); ok {
547 return nil, err
548 }
549
550 var q ghostQuery
551
552 variables := map[string]interface{}{
553 "login": githubv4.String("ghost"),
554 }
555
556 gc := buildClient(gi.conf[core.ConfigKeyToken])
557
558 ctx, cancel := context.WithTimeout(gi.iterator.ctx, defaultTimeout)
559 defer cancel()
560
561 err = gc.Query(ctx, &q, variables)
562 if err != nil {
563 return nil, err
564 }
565
566 var name string
567 if q.User.Name != nil {
568 name = string(*q.User.Name)
569 }
570
571 return repo.NewIdentityRaw(
572 name,
573 "",
574 string(q.User.Login),
575 string(q.User.AvatarUrl),
576 map[string]string{
577 metaKeyGithubLogin: string(q.User.Login),
578 },
579 )
580}
581
582// parseId convert the unusable githubv4.ID (an interface{}) into a string
583func parseId(id githubv4.ID) string {
584 return fmt.Sprintf("%v", id)
585}