1package github
2
3import (
4 "context"
5 "fmt"
6 "strings"
7 "time"
8
9 "github.com/MichaelMure/git-bug/bridge/core"
10 "github.com/MichaelMure/git-bug/bug"
11 "github.com/MichaelMure/git-bug/cache"
12 "github.com/MichaelMure/git-bug/identity"
13 "github.com/MichaelMure/git-bug/util/git"
14 "github.com/shurcooL/githubv4"
15)
16
17const (
18 keyGithubId = "github-id"
19 keyGithubUrl = "github-url"
20 keyGithubLogin = "github-login"
21)
22
23// githubImporter implement the Importer interface
24type githubImporter struct {
25 iterator *iterator
26 conf core.Configuration
27}
28
29func (gi *githubImporter) Init(conf core.Configuration) error {
30 gi.conf = conf
31 gi.iterator = newIterator(conf)
32 return nil
33}
34
35// ImportAll .
36func (gi *githubImporter) ImportAll(repo *cache.RepoCache, since time.Time) error {
37 gi.iterator.since = since
38
39 // Loop over all matching issues
40 for gi.iterator.NextIssue() {
41 issue := gi.iterator.IssueValue()
42
43 fmt.Printf("importing issue: %v %v\n", gi.iterator.count, issue.Title)
44 // get issue edits
45 issueEdits := []userContentEdit{}
46 for gi.iterator.NextIssueEdit() {
47 if issueEdit := gi.iterator.IssueEditValue(); issueEdit.Diff != nil && string(*issueEdit.Diff) != "" {
48 issueEdits = append(issueEdits, issueEdit)
49 }
50 }
51
52 // create issue
53 b, err := gi.ensureIssue(repo, issue, issueEdits)
54 if err != nil {
55 return fmt.Errorf("issue creation: %v", err)
56 }
57
58 // loop over timeline items
59 for gi.iterator.NextTimeline() {
60 item := gi.iterator.TimelineValue()
61
62 // if item is comment
63 if item.Typename == "IssueComment" {
64 // collect all edits
65 commentEdits := []userContentEdit{}
66 for gi.iterator.NextCommentEdit() {
67 if commentEdit := gi.iterator.CommentEditValue(); commentEdit.Diff != nil && string(*commentEdit.Diff) != "" {
68 commentEdits = append(commentEdits, commentEdit)
69 }
70 }
71
72 err := gi.ensureTimelineComment(repo, b, item.IssueComment, commentEdits)
73 if err != nil {
74 return fmt.Errorf("timeline comment creation: %v", err)
75 }
76
77 } else {
78 if err := gi.ensureTimelineItem(repo, b, item); err != nil {
79 return fmt.Errorf("timeline event creation: %v", err)
80 }
81 }
82 }
83
84 // commit bug state
85 if err := b.CommitAsNeeded(); err != nil {
86 return fmt.Errorf("bug commit: %v", err)
87 }
88 }
89
90 if err := gi.iterator.Error(); err != nil {
91 fmt.Printf("import error: %v\n", err)
92 return err
93 }
94
95 fmt.Printf("Successfully imported %v issues from Github\n", gi.iterator.Count())
96 return nil
97}
98
99func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline, issueEdits []userContentEdit) (*cache.BugCache, error) {
100 // ensure issue author
101 author, err := gi.ensurePerson(repo, issue.Author)
102 if err != nil {
103 return nil, err
104 }
105
106 // resolve bug
107 b, err := repo.ResolveBugCreateMetadata(keyGithubUrl, issue.Url.String())
108 if err != nil && err != bug.ErrBugNotExist {
109 return nil, err
110 }
111
112 // if issueEdits is empty
113 if len(issueEdits) == 0 {
114 if err == bug.ErrBugNotExist {
115 // create bug
116 b, err = repo.NewBugRaw(
117 author,
118 issue.CreatedAt.Unix(),
119 issue.Title,
120 cleanupText(string(issue.Body)),
121 nil,
122 map[string]string{
123 keyGithubId: parseId(issue.Id),
124 keyGithubUrl: issue.Url.String(),
125 })
126 if err != nil {
127 return nil, err
128 }
129 }
130
131 } else {
132 // create bug from given issueEdits
133 for i, edit := range issueEdits {
134 if i == 0 && b != nil {
135 continue
136 }
137
138 // if the bug doesn't exist
139 if b == nil {
140 // we create the bug as soon as we have a legit first edition
141 b, err = repo.NewBugRaw(
142 author,
143 issue.CreatedAt.Unix(),
144 issue.Title,
145 cleanupText(string(*edit.Diff)),
146 nil,
147 map[string]string{
148 keyGithubId: parseId(issue.Id),
149 keyGithubUrl: issue.Url.String(),
150 },
151 )
152
153 if err != nil {
154 return nil, err
155 }
156
157 continue
158 }
159
160 // other edits will be added as CommentEdit operations
161 target, err := b.ResolveOperationWithMetadata(keyGithubUrl, issue.Url.String())
162 if err != nil {
163 return nil, err
164 }
165
166 err = gi.ensureCommentEdit(repo, b, target, edit)
167 if err != nil {
168 return nil, err
169 }
170 }
171 }
172
173 return b, nil
174}
175
176func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, item timelineItem) error {
177 fmt.Printf("import event item: %s\n", item.Typename)
178
179 switch item.Typename {
180 case "IssueComment":
181
182 case "LabeledEvent":
183 id := parseId(item.LabeledEvent.Id)
184 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
185 if err != cache.ErrNoMatchingOp {
186 return err
187 }
188 author, err := gi.ensurePerson(repo, item.LabeledEvent.Actor)
189 if err != nil {
190 return err
191 }
192 _, err = b.ForceChangeLabelsRaw(
193 author,
194 item.LabeledEvent.CreatedAt.Unix(),
195 []string{
196 string(item.LabeledEvent.Label.Name),
197 },
198 nil,
199 map[string]string{keyGithubId: id},
200 )
201
202 return err
203
204 case "UnlabeledEvent":
205 id := parseId(item.UnlabeledEvent.Id)
206 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
207 if err != cache.ErrNoMatchingOp {
208 return err
209 }
210 author, err := gi.ensurePerson(repo, item.UnlabeledEvent.Actor)
211 if err != nil {
212 return err
213 }
214
215 _, err = b.ForceChangeLabelsRaw(
216 author,
217 item.UnlabeledEvent.CreatedAt.Unix(),
218 nil,
219 []string{
220 string(item.UnlabeledEvent.Label.Name),
221 },
222 map[string]string{keyGithubId: id},
223 )
224 return err
225
226 case "ClosedEvent":
227 id := parseId(item.ClosedEvent.Id)
228 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
229 if err != cache.ErrNoMatchingOp {
230 return err
231 }
232 author, err := gi.ensurePerson(repo, item.ClosedEvent.Actor)
233 if err != nil {
234 return err
235 }
236 _, err = b.CloseRaw(
237 author,
238 item.ClosedEvent.CreatedAt.Unix(),
239 map[string]string{keyGithubId: id},
240 )
241 return err
242
243 case "ReopenedEvent":
244 id := parseId(item.ReopenedEvent.Id)
245 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
246 if err != cache.ErrNoMatchingOp {
247 return err
248 }
249 author, err := gi.ensurePerson(repo, item.ReopenedEvent.Actor)
250 if err != nil {
251 return err
252 }
253 _, err = b.OpenRaw(
254 author,
255 item.ReopenedEvent.CreatedAt.Unix(),
256 map[string]string{keyGithubId: id},
257 )
258 return err
259
260 case "RenamedTitleEvent":
261 id := parseId(item.RenamedTitleEvent.Id)
262 _, err := b.ResolveOperationWithMetadata(keyGithubId, id)
263 if err != cache.ErrNoMatchingOp {
264 return err
265 }
266 author, err := gi.ensurePerson(repo, item.RenamedTitleEvent.Actor)
267 if err != nil {
268 return err
269 }
270 _, err = b.SetTitleRaw(
271 author,
272 item.RenamedTitleEvent.CreatedAt.Unix(),
273 string(item.RenamedTitleEvent.CurrentTitle),
274 map[string]string{keyGithubId: id},
275 )
276 return err
277
278 default:
279 fmt.Printf("ignore event: %v\n", item.Typename)
280 }
281
282 return nil
283}
284
285func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache.BugCache, item issueComment, edits []userContentEdit) error {
286 // ensure person
287 author, err := gi.ensurePerson(repo, item.Author)
288 if err != nil {
289 return err
290 }
291
292 var target git.Hash
293 target, err = b.ResolveOperationWithMetadata(keyGithubId, parseId(item.Id))
294 if err != nil && err != cache.ErrNoMatchingOp {
295 // real error
296 return err
297 }
298 // if no edits are given we create the comment
299 if len(edits) == 0 {
300
301 // if comment doesn't exist
302 if err == cache.ErrNoMatchingOp {
303
304 // add comment operation
305 op, err := b.AddCommentRaw(
306 author,
307 item.CreatedAt.Unix(),
308 cleanupText(string(item.Body)),
309 nil,
310 map[string]string{
311 keyGithubId: parseId(item.Id),
312 keyGithubUrl: parseId(item.Url.String()),
313 },
314 )
315 if err != nil {
316 return err
317 }
318
319 // set hash
320 target, err = op.Hash()
321 if err != nil {
322 return err
323 }
324 }
325 } else {
326 for i, edit := range item.UserContentEdits.Nodes {
327 if i == 0 && target != "" {
328 continue
329 }
330
331 // ensure editor identity
332 editor, err := gi.ensurePerson(repo, edit.Editor)
333 if err != nil {
334 return err
335 }
336
337 // create comment when target is empty
338 if target == "" {
339 op, err := b.AddCommentRaw(
340 editor,
341 edit.CreatedAt.Unix(),
342 cleanupText(string(*edit.Diff)),
343 nil,
344 map[string]string{
345 keyGithubId: parseId(item.Id),
346 keyGithubUrl: item.Url.String(),
347 },
348 )
349 if err != nil {
350 return err
351 }
352
353 // set hash
354 target, err = op.Hash()
355 if err != nil {
356 return err
357 }
358
359 continue
360 }
361
362 err = gi.ensureCommentEdit(repo, b, target, edit)
363 if err != nil {
364 return err
365 }
366 }
367 }
368 return nil
369}
370
371func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugCache, target git.Hash, edit userContentEdit) error {
372 _, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(edit.Id))
373 if err == nil {
374 // already imported
375 return nil
376 }
377 if err != cache.ErrNoMatchingOp {
378 // real error
379 return err
380 }
381
382 fmt.Println("import edition")
383
384 editor, err := gi.ensurePerson(repo, edit.Editor)
385 if err != nil {
386 return err
387 }
388
389 switch {
390 case edit.DeletedAt != nil:
391 // comment deletion, not supported yet
392 fmt.Println("comment deletion ....")
393
394 case edit.DeletedAt == nil:
395
396 // comment edition
397 _, err := b.EditCommentRaw(
398 editor,
399 edit.CreatedAt.Unix(),
400 target,
401 cleanupText(string(*edit.Diff)),
402 map[string]string{
403 keyGithubId: parseId(edit.Id),
404 },
405 )
406
407 if err != nil {
408 return err
409 }
410 }
411
412 return nil
413}
414
415// ensurePerson create a bug.Person from the Github data
416func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
417 // When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
418 // in it's UI. So we need a special case to get it.
419 if actor == nil {
420 return gi.getGhost(repo)
421 }
422
423 // Look first in the cache
424 i, err := repo.ResolveIdentityImmutableMetadata(keyGithubLogin, string(actor.Login))
425 if err == nil {
426 return i, nil
427 }
428 if _, ok := err.(identity.ErrMultipleMatch); ok {
429 return nil, err
430 }
431
432 var name string
433 var email string
434
435 switch actor.Typename {
436 case "User":
437 if actor.User.Name != nil {
438 name = string(*(actor.User.Name))
439 }
440 email = string(actor.User.Email)
441 case "Organization":
442 if actor.Organization.Name != nil {
443 name = string(*(actor.Organization.Name))
444 }
445 if actor.Organization.Email != nil {
446 email = string(*(actor.Organization.Email))
447 }
448 case "Bot":
449 }
450
451 return repo.NewIdentityRaw(
452 name,
453 email,
454 string(actor.Login),
455 string(actor.AvatarUrl),
456 map[string]string{
457 keyGithubLogin: string(actor.Login),
458 },
459 )
460}
461
462func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, error) {
463 // Look first in the cache
464 i, err := repo.ResolveIdentityImmutableMetadata(keyGithubLogin, "ghost")
465 if err == nil {
466 return i, nil
467 }
468 if _, ok := err.(identity.ErrMultipleMatch); ok {
469 return nil, err
470 }
471
472 var q userQuery
473
474 variables := map[string]interface{}{
475 "login": githubv4.String("ghost"),
476 }
477
478 err = gi.iterator.gc.Query(context.TODO(), &q, variables)
479 if err != nil {
480 return nil, err
481 }
482
483 var name string
484 if q.User.Name != nil {
485 name = string(*q.User.Name)
486 }
487
488 return repo.NewIdentityRaw(
489 name,
490 string(q.User.Email),
491 string(q.User.Login),
492 string(q.User.AvatarUrl),
493 map[string]string{
494 keyGithubLogin: string(q.User.Login),
495 },
496 )
497}
498
499// parseId convert the unusable githubv4.ID (an interface{}) into a string
500func parseId(id githubv4.ID) string {
501 return fmt.Sprintf("%v", id)
502}
503
504func cleanupText(text string) string {
505 // windows new line, Github, really ?
506 text = strings.Replace(text, "\r\n", "\n", -1)
507
508 // trim extra new line not displayed in the github UI but still present in the data
509 return strings.TrimSpace(text)
510}
511
512func reverseEdits(edits []userContentEdit) []userContentEdit {
513 for i, j := 0, len(edits)-1; i < j; i, j = i+1, j-1 {
514 edits[i], edits[j] = edits[j], edits[i]
515 }
516 return edits
517}