1package github
2
3import (
4 "context"
5 "fmt"
6 "strings"
7 "time"
8
9 "github.com/MichaelMure/git-bug/bridge/core"
10 "github.com/MichaelMure/git-bug/bug"
11 "github.com/MichaelMure/git-bug/cache"
12 "github.com/MichaelMure/git-bug/identity"
13 "github.com/MichaelMure/git-bug/util/git"
14 "github.com/shurcooL/githubv4"
15)
16
17const (
18 keyGithubId = "github-id"
19 keyGithubUrl = "github-url"
20 keyGithubLogin = "github-login"
21)
22
23// githubImporter implement the Importer interface
24type githubImporter struct {
25 conf core.Configuration
26}
27
28func (gi *githubImporter) Init(conf core.Configuration) error {
29 gi.conf = conf
30 return nil
31}
32
33// ImportAll .
34func (gi *githubImporter) ImportAll(repo *cache.RepoCache, since time.Time) error {
35 iterator := NewIterator(gi.conf[keyUser], gi.conf[keyProject], gi.conf[keyToken], since)
36
37 // Loop over all matching issues
38 for iterator.NextIssue() {
39 issue := iterator.IssueValue()
40
41 fmt.Printf("importing issue: %v %v\n", iterator.importedIssues, issue.Title)
42 // get issue edits
43 issueEdits := []userContentEdit{}
44 for iterator.NextIssueEdit() {
45 // issueEdit.Diff == nil happen if the event is older than early 2018, Github doesn't have the data before that.
46 // Best we can do is to ignore the event.
47 if issueEdit := iterator.IssueEditValue(); issueEdit.Diff != nil && string(*issueEdit.Diff) != "" {
48 issueEdits = append(issueEdits, issueEdit)
49 }
50 }
51
52 // create issue
53 b, err := gi.ensureIssue(repo, issue, issueEdits)
54 if err != nil {
55 return fmt.Errorf("issue creation: %v", err)
56 }
57
58 // loop over timeline items
59 for iterator.NextTimeline() {
60 item := iterator.TimelineValue()
61
62 // if item is comment
63 if item.Typename == "IssueComment" {
64 // collect all edits
65 commentEdits := []userContentEdit{}
66 for iterator.NextCommentEdit() {
67 if commentEdit := iterator.CommentEditValue(); commentEdit.Diff != nil && string(*commentEdit.Diff) != "" {
68 commentEdits = append(commentEdits, commentEdit)
69 }
70 }
71
72 err := gi.ensureTimelineComment(repo, b, item.IssueComment, commentEdits)
73 if err != nil {
74 return fmt.Errorf("timeline comment creation: %v", err)
75 }
76
77 } else {
78 if err := gi.ensureTimelineItem(repo, b, item); err != nil {
79 return fmt.Errorf("timeline event creation: %v", err)
80 }
81 }
82 }
83
84 // commit bug state
85 if err := b.CommitAsNeeded(); err != nil {
86 return fmt.Errorf("bug commit: %v", err)
87 }
88 }
89
90 if err := iterator.Error(); err != nil {
91 fmt.Printf("import error: %v\n", err)
92 return err
93 }
94
95 fmt.Printf("Successfully imported %v issues from Github\n", iterator.ImportedIssues())
96 return nil
97}
98
99func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline, issueEdits []userContentEdit) (*cache.BugCache, error) {
100 // ensure issue author
101 author, err := gi.ensurePerson(repo, issue.Author)
102 if err != nil {
103 return nil, err
104 }
105
106 // resolve bug
107 b, err := repo.ResolveBugCreateMetadata(keyGithubUrl, issue.Url.String())
108 if err != nil && err != bug.ErrBugNotExist {
109 return nil, err
110 }
111
112 // if issueEdits is empty
113 if len(issueEdits) == 0 {
114 if err == bug.ErrBugNotExist {
115 // create bug
116 b, err = repo.NewBugRaw(
117 author,
118 issue.CreatedAt.Unix(),
119 issue.Title,
120 cleanupText(string(issue.Body)),
121 nil,
122 map[string]string{
123 keyGithubId: parseId(issue.Id),
124 keyGithubUrl: issue.Url.String(),
125 })
126 if err != nil {
127 return nil, err
128 }
129 }
130
131 } else {
132 // create bug from given issueEdits
133 for i, edit := range issueEdits {
134 if i == 0 && b != nil {
135 // The first edit in the github result is the issue creation itself, we already have that
136 continue
137 }
138
139 // if the bug doesn't exist
140 if b == nil {
141 // we create the bug as soon as we have a legit first edition
142 b, err = repo.NewBugRaw(
143 author,
144 issue.CreatedAt.Unix(),
145 issue.Title,
146 cleanupText(string(*edit.Diff)),
147 nil,
148 map[string]string{
149 keyGithubId: parseId(issue.Id),
150 keyGithubUrl: issue.Url.String(),
151 },
152 )
153
154 if err != nil {
155 return nil, err
156 }
157
158 continue
159 }
160
161 // other edits will be added as CommentEdit operations
162 target, err := b.ResolveOperationWithMetadata(keyGithubUrl, issue.Url.String())
163 if err != nil {
164 return nil, err
165 }
166
167 err = gi.ensureCommentEdit(repo, b, target, edit)
168 if err != nil {
169 return nil, err
170 }
171 }
172 }
173
174 return b, nil
175}
176
177func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, item timelineItem) error {
178 fmt.Printf("import event item: %s\n", item.Typename)
179
180 switch item.Typename {
181 case "IssueComment":
182
183 case "LabeledEvent":
184 id := parseId(item.LabeledEvent.Id)
185 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
186 if err != cache.ErrNoMatchingOp {
187 return err
188 }
189 author, err := gi.ensurePerson(repo, item.LabeledEvent.Actor)
190 if err != nil {
191 return err
192 }
193 _, err = b.ForceChangeLabelsRaw(
194 author,
195 item.LabeledEvent.CreatedAt.Unix(),
196 []string{
197 string(item.LabeledEvent.Label.Name),
198 },
199 nil,
200 map[string]string{keyGithubId: id},
201 )
202
203 return err
204
205 case "UnlabeledEvent":
206 id := parseId(item.UnlabeledEvent.Id)
207 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
208 if err != cache.ErrNoMatchingOp {
209 return err
210 }
211 author, err := gi.ensurePerson(repo, item.UnlabeledEvent.Actor)
212 if err != nil {
213 return err
214 }
215
216 _, err = b.ForceChangeLabelsRaw(
217 author,
218 item.UnlabeledEvent.CreatedAt.Unix(),
219 nil,
220 []string{
221 string(item.UnlabeledEvent.Label.Name),
222 },
223 map[string]string{keyGithubId: id},
224 )
225 return err
226
227 case "ClosedEvent":
228 id := parseId(item.ClosedEvent.Id)
229 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
230 if err != cache.ErrNoMatchingOp {
231 return err
232 }
233 author, err := gi.ensurePerson(repo, item.ClosedEvent.Actor)
234 if err != nil {
235 return err
236 }
237 _, err = b.CloseRaw(
238 author,
239 item.ClosedEvent.CreatedAt.Unix(),
240 map[string]string{keyGithubId: id},
241 )
242 return err
243
244 case "ReopenedEvent":
245 id := parseId(item.ReopenedEvent.Id)
246 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
247 if err != cache.ErrNoMatchingOp {
248 return err
249 }
250 author, err := gi.ensurePerson(repo, item.ReopenedEvent.Actor)
251 if err != nil {
252 return err
253 }
254 _, err = b.OpenRaw(
255 author,
256 item.ReopenedEvent.CreatedAt.Unix(),
257 map[string]string{keyGithubId: id},
258 )
259 return err
260
261 case "RenamedTitleEvent":
262 id := parseId(item.RenamedTitleEvent.Id)
263 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
264 if err != cache.ErrNoMatchingOp {
265 return err
266 }
267 author, err := gi.ensurePerson(repo, item.RenamedTitleEvent.Actor)
268 if err != nil {
269 return err
270 }
271 _, err = b.SetTitleRaw(
272 author,
273 item.RenamedTitleEvent.CreatedAt.Unix(),
274 string(item.RenamedTitleEvent.CurrentTitle),
275 map[string]string{keyGithubId: id},
276 )
277 return err
278
279 default:
280 fmt.Printf("ignore event: %v\n", item.Typename)
281 }
282
283 return nil
284}
285
286func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache.BugCache, item issueComment, edits []userContentEdit) error {
287 // ensure person
288 author, err := gi.ensurePerson(repo, item.Author)
289 if err != nil {
290 return err
291 }
292
293 var target git.Hash
294 target, err = b.ResolveOperationWithMetadata(keyGithubId, parseId(item.Id))
295 if err != nil && err != cache.ErrNoMatchingOp {
296 // real error
297 return err
298 }
299 // if no edits are given we create the comment
300 if len(edits) == 0 {
301
302 // if comment doesn't exist
303 if err == cache.ErrNoMatchingOp {
304
305 // add comment operation
306 op, err := b.AddCommentRaw(
307 author,
308 item.CreatedAt.Unix(),
309 cleanupText(string(item.Body)),
310 nil,
311 map[string]string{
312 keyGithubId: parseId(item.Id),
313 keyGithubUrl: parseId(item.Url.String()),
314 },
315 )
316 if err != nil {
317 return err
318 }
319
320 // set hash
321 target, err = op.Hash()
322 if err != nil {
323 return err
324 }
325 }
326 } else {
327 for i, edit := range item.UserContentEdits.Nodes {
328 if i == 0 && target != "" {
329 // The first edit in the github result is the comment creation itself, we already have that
330 continue
331 }
332
333 // ensure editor identity
334 editor, err := gi.ensurePerson(repo, edit.Editor)
335 if err != nil {
336 return err
337 }
338
339 // create comment when target is empty
340 if target == "" {
341 op, err := b.AddCommentRaw(
342 editor,
343 edit.CreatedAt.Unix(),
344 cleanupText(string(*edit.Diff)),
345 nil,
346 map[string]string{
347 keyGithubId: parseId(item.Id),
348 keyGithubUrl: item.Url.String(),
349 },
350 )
351 if err != nil {
352 return err
353 }
354
355 // set hash
356 target, err = op.Hash()
357 if err != nil {
358 return err
359 }
360
361 continue
362 }
363
364 err = gi.ensureCommentEdit(repo, b, target, edit)
365 if err != nil {
366 return err
367 }
368 }
369 }
370 return nil
371}
372
373func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugCache, target git.Hash, edit userContentEdit) error {
374 _, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(edit.Id))
375 if err == nil {
376 // already imported
377 return nil
378 }
379 if err != cache.ErrNoMatchingOp {
380 // real error
381 return err
382 }
383
384 fmt.Println("import edition")
385
386 editor, err := gi.ensurePerson(repo, edit.Editor)
387 if err != nil {
388 return err
389 }
390
391 switch {
392 case edit.DeletedAt != nil:
393 // comment deletion, not supported yet
394 fmt.Println("comment deletion is not supported yet")
395
396 case edit.DeletedAt == nil:
397
398 // comment edition
399 _, err := b.EditCommentRaw(
400 editor,
401 edit.CreatedAt.Unix(),
402 target,
403 cleanupText(string(*edit.Diff)),
404 map[string]string{
405 keyGithubId: parseId(edit.Id),
406 },
407 )
408
409 if err != nil {
410 return err
411 }
412 }
413
414 return nil
415}
416
417// ensurePerson create a bug.Person from the Github data
418func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
419 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
420 // in it's UI. So we need a special case to get it.
421 if actor == nil {
422 return gi.getGhost(repo)
423 }
424
425 // Look first in the cache
426 i, err := repo.ResolveIdentityImmutableMetadata(keyGithubLogin, string(actor.Login))
427 if err == nil {
428 return i, nil
429 }
430 if _, ok := err.(identity.ErrMultipleMatch); ok {
431 return nil, err
432 }
433
434 var name string
435 var email string
436
437 switch actor.Typename {
438 case "User":
439 if actor.User.Name != nil {
440 name = string(*(actor.User.Name))
441 }
442 email = string(actor.User.Email)
443 case "Organization":
444 if actor.Organization.Name != nil {
445 name = string(*(actor.Organization.Name))
446 }
447 if actor.Organization.Email != nil {
448 email = string(*(actor.Organization.Email))
449 }
450 case "Bot":
451 }
452
453 return repo.NewIdentityRaw(
454 name,
455 email,
456 string(actor.Login),
457 string(actor.AvatarUrl),
458 map[string]string{
459 keyGithubLogin: string(actor.Login),
460 },
461 )
462}
463
464func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, error) {
465 // Look first in the cache
466 i, err := repo.ResolveIdentityImmutableMetadata(keyGithubLogin, "ghost")
467 if err == nil {
468 return i, nil
469 }
470 if _, ok := err.(identity.ErrMultipleMatch); ok {
471 return nil, err
472 }
473
474 var q userQuery
475
476 variables := map[string]interface{}{
477 "login": githubv4.String("ghost"),
478 }
479
480 gc := buildClient(gi.conf[keyToken])
481
482 err = gc.Query(context.TODO(), &q, variables)
483 if err != nil {
484 return nil, err
485 }
486
487 var name string
488 if q.User.Name != nil {
489 name = string(*q.User.Name)
490 }
491
492 return repo.NewIdentityRaw(
493 name,
494 string(q.User.Email),
495 string(q.User.Login),
496 string(q.User.AvatarUrl),
497 map[string]string{
498 keyGithubLogin: string(q.User.Login),
499 },
500 )
501}
502
503// parseId convert the unusable githubv4.ID (an interface{}) into a string
504func parseId(id githubv4.ID) string {
505 return fmt.Sprintf("%v", id)
506}
507
508func cleanupText(text string) string {
509 // windows new line, Github, really ?
510 text = strings.Replace(text, "\r\n", "\n", -1)
511
512 // trim extra new line not displayed in the github UI but still present in the data
513 return strings.TrimSpace(text)
514}
515
516func reverseEdits(edits []userContentEdit) []userContentEdit {
517 for i, j := 0, len(edits)-1; i < j; i, j = i+1, j-1 {
518 edits[i], edits[j] = edits[j], edits[i]
519 }
520 return edits
521}