1package github
  2
  3import (
  4	"context"
  5	"fmt"
  6	"time"
  7
  8	"github.com/shurcooL/githubv4"
  9
 10	"github.com/MichaelMure/git-bug/bridge/core"
 11	"github.com/MichaelMure/git-bug/bug"
 12	"github.com/MichaelMure/git-bug/cache"
 13	"github.com/MichaelMure/git-bug/entity"
 14	"github.com/MichaelMure/git-bug/util/text"
 15)
 16
 17const (
 18	keyOrigin      = "origin"
 19	keyGithubId    = "github-id"
 20	keyGithubUrl   = "github-url"
 21	keyGithubLogin = "github-login"
 22)
 23
 24// githubImporter implement the Importer interface
 25type githubImporter struct {
 26	conf core.Configuration
 27
 28	// iterator
 29	iterator *iterator
 30
 31	// send only channel
 32	out chan<- core.ImportResult
 33}
 34
 35func (gi *githubImporter) Init(conf core.Configuration) error {
 36	gi.conf = conf
 37	return nil
 38}
 39
 40// ImportAll iterate over all the configured repository issues and ensure the creation of the
 41// missing issues / timeline items / edits / label events ...
 42func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
 43	gi.iterator = NewIterator(ctx, 10, gi.conf[keyOwner], gi.conf[keyProject], gi.conf[keyToken], since)
 44	out := make(chan core.ImportResult)
 45	gi.out = out
 46
 47	go func() {
 48		defer close(gi.out)
 49
 50		// Loop over all matching issues
 51		for gi.iterator.NextIssue() {
 52			issue := gi.iterator.IssueValue()
 53			// create issue
 54			b, err := gi.ensureIssue(repo, issue)
 55			if err != nil {
 56				err := fmt.Errorf("issue creation: %v", err)
 57				out <- core.NewImportError(err, "")
 58				return
 59			}
 60
 61			// loop over timeline items
 62			for gi.iterator.NextTimelineItem() {
 63				item := gi.iterator.TimelineItemValue()
 64				if err := gi.ensureTimelineItem(repo, b, item); err != nil {
 65					err := fmt.Errorf("timeline item creation: %v", err)
 66					out <- core.NewImportError(err, "")
 67					return
 68				}
 69			}
 70
 71			// commit bug state
 72			if err := b.CommitAsNeeded(); err != nil {
 73				err = fmt.Errorf("bug commit: %v", err)
 74				out <- core.NewImportError(err, "")
 75				return
 76			}
 77		}
 78
 79		if err := gi.iterator.Error(); err != nil && err != context.Canceled {
 80			gi.out <- core.NewImportError(err, "")
 81		}
 82	}()
 83
 84	return out, nil
 85}
 86
 87func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline) (*cache.BugCache, error) {
 88	// ensure issue author
 89	author, err := gi.ensurePerson(repo, issue.Author)
 90	if err != nil {
 91		return nil, err
 92	}
 93
 94	// resolve bug
 95	b, err := repo.ResolveBugCreateMetadata(keyGithubUrl, issue.Url.String())
 96	if err != nil && err != bug.ErrBugNotExist {
 97		return nil, err
 98	}
 99
100	// get issue edits
101	var issueEdits []userContentEdit
102	for gi.iterator.NextIssueEdit() {
103		issueEdits = append(issueEdits, gi.iterator.IssueEditValue())
104	}
105
106	// if issueEdits is empty
107	if len(issueEdits) == 0 {
108		if err == bug.ErrBugNotExist {
109			cleanText, err := text.Cleanup(string(issue.Body))
110			if err != nil {
111				return nil, err
112			}
113
114			// create bug
115			b, _, err = repo.NewBugRaw(
116				author,
117				issue.CreatedAt.Unix(),
118				issue.Title,
119				cleanText,
120				nil,
121				map[string]string{
122					keyOrigin:    target,
123					keyGithubId:  parseId(issue.Id),
124					keyGithubUrl: issue.Url.String(),
125				})
126			if err != nil {
127				return nil, err
128			}
129
130			// importing a new bug
131			gi.out <- core.NewImportBug(b.Id())
132		} else {
133			gi.out <- core.NewImportNothing("", "bug already imported")
134		}
135
136	} else {
137		// create bug from given issueEdits
138		for i, edit := range issueEdits {
139			if i == 0 && b != nil {
140				// The first edit in the github result is the issue creation itself, we already have that
141				gi.out <- core.NewImportNothing("", "bug already imported")
142				continue
143			}
144
145			cleanText, err := text.Cleanup(string(*edit.Diff))
146			if err != nil {
147				return nil, err
148			}
149
150			// if the bug doesn't exist
151			if b == nil {
152				// we create the bug as soon as we have a legit first edition
153				b, _, err = repo.NewBugRaw(
154					author,
155					issue.CreatedAt.Unix(),
156					issue.Title,
157					cleanText,
158					nil,
159					map[string]string{
160						keyOrigin:    target,
161						keyGithubId:  parseId(issue.Id),
162						keyGithubUrl: issue.Url.String(),
163					},
164				)
165
166				if err != nil {
167					return nil, err
168				}
169
170				// importing a new bug
171				gi.out <- core.NewImportBug(b.Id())
172				continue
173			}
174
175			// other edits will be added as CommentEdit operations
176			target, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(issue.Id))
177			if err != nil {
178				return nil, err
179			}
180
181			err = gi.ensureCommentEdit(repo, b, target, edit)
182			if err != nil {
183				return nil, err
184			}
185		}
186	}
187
188	return b, nil
189}
190
191func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, item timelineItem) error {
192
193	switch item.Typename {
194	case "IssueComment":
195		// collect all comment edits
196		var commentEdits []userContentEdit
197		for gi.iterator.NextCommentEdit() {
198			commentEdits = append(commentEdits, gi.iterator.CommentEditValue())
199		}
200
201		// ensureTimelineComment send import events over out chanel
202		err := gi.ensureTimelineComment(repo, b, item.IssueComment, commentEdits)
203		if err != nil {
204			return fmt.Errorf("timeline comment creation: %v", err)
205		}
206
207	case "LabeledEvent":
208		id := parseId(item.LabeledEvent.Id)
209		_, err := b.ResolveOperationWithMetadata(keyGithubId, id)
210		if err == nil {
211			reason := fmt.Sprintf("operation already imported: %v", item.Typename)
212			gi.out <- core.NewImportNothing("", reason)
213			return nil
214		}
215
216		if err != cache.ErrNoMatchingOp {
217			return err
218		}
219		author, err := gi.ensurePerson(repo, item.LabeledEvent.Actor)
220		if err != nil {
221			return err
222		}
223		op, err := b.ForceChangeLabelsRaw(
224			author,
225			item.LabeledEvent.CreatedAt.Unix(),
226			[]string{
227				string(item.LabeledEvent.Label.Name),
228			},
229			nil,
230			map[string]string{keyGithubId: id},
231		)
232		if err != nil {
233			return err
234		}
235
236		gi.out <- core.NewImportLabelChange(op.Id())
237		return nil
238
239	case "UnlabeledEvent":
240		id := parseId(item.UnlabeledEvent.Id)
241		_, err := b.ResolveOperationWithMetadata(keyGithubId, id)
242		if err == nil {
243			reason := fmt.Sprintf("operation already imported: %v", item.Typename)
244			gi.out <- core.NewImportNothing("", reason)
245			return nil
246		}
247		if err != cache.ErrNoMatchingOp {
248			return err
249		}
250		author, err := gi.ensurePerson(repo, item.UnlabeledEvent.Actor)
251		if err != nil {
252			return err
253		}
254
255		op, err := b.ForceChangeLabelsRaw(
256			author,
257			item.UnlabeledEvent.CreatedAt.Unix(),
258			nil,
259			[]string{
260				string(item.UnlabeledEvent.Label.Name),
261			},
262			map[string]string{keyGithubId: id},
263		)
264		if err != nil {
265			return err
266		}
267
268		gi.out <- core.NewImportLabelChange(op.Id())
269		return nil
270
271	case "ClosedEvent":
272		id := parseId(item.ClosedEvent.Id)
273		_, err := b.ResolveOperationWithMetadata(keyGithubId, id)
274		if err != cache.ErrNoMatchingOp {
275			return err
276		}
277		if err == nil {
278			reason := fmt.Sprintf("operation already imported: %v", item.Typename)
279			gi.out <- core.NewImportNothing("", reason)
280			return nil
281		}
282		author, err := gi.ensurePerson(repo, item.ClosedEvent.Actor)
283		if err != nil {
284			return err
285		}
286		op, err := b.CloseRaw(
287			author,
288			item.ClosedEvent.CreatedAt.Unix(),
289			map[string]string{keyGithubId: id},
290		)
291
292		if err != nil {
293			return err
294		}
295
296		gi.out <- core.NewImportStatusChange(op.Id())
297		return nil
298
299	case "ReopenedEvent":
300		id := parseId(item.ReopenedEvent.Id)
301		_, err := b.ResolveOperationWithMetadata(keyGithubId, id)
302		if err != cache.ErrNoMatchingOp {
303			return err
304		}
305		if err == nil {
306			reason := fmt.Sprintf("operation already imported: %v", item.Typename)
307			gi.out <- core.NewImportNothing("", reason)
308			return nil
309		}
310		author, err := gi.ensurePerson(repo, item.ReopenedEvent.Actor)
311		if err != nil {
312			return err
313		}
314		op, err := b.OpenRaw(
315			author,
316			item.ReopenedEvent.CreatedAt.Unix(),
317			map[string]string{keyGithubId: id},
318		)
319
320		if err != nil {
321			return err
322		}
323
324		gi.out <- core.NewImportStatusChange(op.Id())
325		return nil
326
327	case "RenamedTitleEvent":
328		id := parseId(item.RenamedTitleEvent.Id)
329		_, err := b.ResolveOperationWithMetadata(keyGithubId, id)
330		if err != cache.ErrNoMatchingOp {
331			return err
332		}
333		if err == nil {
334			reason := fmt.Sprintf("operation already imported: %v", item.Typename)
335			gi.out <- core.NewImportNothing("", reason)
336			return nil
337		}
338		author, err := gi.ensurePerson(repo, item.RenamedTitleEvent.Actor)
339		if err != nil {
340			return err
341		}
342		op, err := b.SetTitleRaw(
343			author,
344			item.RenamedTitleEvent.CreatedAt.Unix(),
345			string(item.RenamedTitleEvent.CurrentTitle),
346			map[string]string{keyGithubId: id},
347		)
348		if err != nil {
349			return err
350		}
351
352		gi.out <- core.NewImportTitleEdition(op.Id())
353		return nil
354
355	default:
356		reason := fmt.Sprintf("ignoring timeline type: %v", item.Typename)
357		gi.out <- core.NewImportNothing("", reason)
358	}
359
360	return nil
361}
362
363func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache.BugCache, item issueComment, edits []userContentEdit) error {
364	// ensure person
365	author, err := gi.ensurePerson(repo, item.Author)
366	if err != nil {
367		return err
368	}
369
370	targetOpID, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(item.Id))
371	if err == nil {
372		reason := fmt.Sprintf("comment already imported")
373		gi.out <- core.NewImportNothing("", reason)
374	} else if err != cache.ErrNoMatchingOp {
375		// real error
376		return err
377	}
378
379	// if no edits are given we create the comment
380	if len(edits) == 0 {
381		if err == cache.ErrNoMatchingOp {
382			cleanText, err := text.Cleanup(string(item.Body))
383			if err != nil {
384				return err
385			}
386
387			// add comment operation
388			op, err := b.AddCommentRaw(
389				author,
390				item.CreatedAt.Unix(),
391				cleanText,
392				nil,
393				map[string]string{
394					keyGithubId:  parseId(item.Id),
395					keyGithubUrl: parseId(item.Url.String()),
396				},
397			)
398			if err != nil {
399				return err
400			}
401
402			gi.out <- core.NewImportComment(op.Id())
403		}
404
405	} else {
406		for i, edit := range edits {
407			if i == 0 && targetOpID != "" {
408				// The first edit in the github result is the comment creation itself, we already have that
409				gi.out <- core.NewImportNothing("", "comment already imported")
410				continue
411			}
412
413			// ensure editor identity
414			editor, err := gi.ensurePerson(repo, edit.Editor)
415			if err != nil {
416				return err
417			}
418
419			// create comment when target is empty
420			if targetOpID == "" {
421				cleanText, err := text.Cleanup(string(*edit.Diff))
422				if err != nil {
423					return err
424				}
425
426				op, err := b.AddCommentRaw(
427					editor,
428					edit.CreatedAt.Unix(),
429					cleanText,
430					nil,
431					map[string]string{
432						keyGithubId:  parseId(item.Id),
433						keyGithubUrl: item.Url.String(),
434					},
435				)
436				if err != nil {
437					return err
438				}
439
440				// set target for the nexr edit now that the comment is created
441				targetOpID = op.Id()
442				continue
443			}
444
445			err = gi.ensureCommentEdit(repo, b, targetOpID, edit)
446			if err != nil {
447				return err
448			}
449		}
450	}
451	return nil
452}
453
454func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugCache, target entity.Id, edit userContentEdit) error {
455	_, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(edit.Id))
456	if err == nil {
457		gi.out <- core.NewImportNothing(b.Id(), "edition already imported")
458		return nil
459	}
460	if err != cache.ErrNoMatchingOp {
461		// real error
462		return err
463	}
464
465	editor, err := gi.ensurePerson(repo, edit.Editor)
466	if err != nil {
467		return err
468	}
469
470	switch {
471	case edit.DeletedAt != nil:
472		// comment deletion, not supported yet
473		gi.out <- core.NewImportNothing(b.Id(), "comment deletion is not supported yet")
474
475	case edit.DeletedAt == nil:
476
477		cleanText, err := text.Cleanup(string(*edit.Diff))
478		if err != nil {
479			return err
480		}
481
482		// comment edition
483		op, err := b.EditCommentRaw(
484			editor,
485			edit.CreatedAt.Unix(),
486			target,
487			cleanText,
488			map[string]string{
489				keyGithubId: parseId(edit.Id),
490			},
491		)
492
493		if err != nil {
494			return err
495		}
496
497		gi.out <- core.NewImportCommentEdition(op.Id())
498	}
499
500	return nil
501}
502
503// ensurePerson create a bug.Person from the Github data
504func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
505	// When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
506	// in it's UI. So we need a special case to get it.
507	if actor == nil {
508		return gi.getGhost(repo)
509	}
510
511	// Look first in the cache
512	i, err := repo.ResolveIdentityImmutableMetadata(keyGithubLogin, string(actor.Login))
513	if err == nil {
514		return i, nil
515	}
516	if _, ok := err.(entity.ErrMultipleMatch); ok {
517		return nil, err
518	}
519
520	// importing a new identity
521
522	var name string
523	var email string
524
525	switch actor.Typename {
526	case "User":
527		if actor.User.Name != nil {
528			name = string(*(actor.User.Name))
529		}
530		email = string(actor.User.Email)
531	case "Organization":
532		if actor.Organization.Name != nil {
533			name = string(*(actor.Organization.Name))
534		}
535		if actor.Organization.Email != nil {
536			email = string(*(actor.Organization.Email))
537		}
538	case "Bot":
539	}
540
541	i, err = repo.NewIdentityRaw(
542		name,
543		email,
544		string(actor.Login),
545		string(actor.AvatarUrl),
546		map[string]string{
547			keyGithubLogin: string(actor.Login),
548		},
549	)
550
551	if err != nil {
552		return nil, err
553	}
554
555	gi.out <- core.NewImportIdentity(i.Id())
556	return i, nil
557}
558
559func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, error) {
560	// Look first in the cache
561	i, err := repo.ResolveIdentityImmutableMetadata(keyGithubLogin, "ghost")
562	if err == nil {
563		return i, nil
564	}
565	if _, ok := err.(entity.ErrMultipleMatch); ok {
566		return nil, err
567	}
568
569	var q ghostQuery
570
571	variables := map[string]interface{}{
572		"login": githubv4.String("ghost"),
573	}
574
575	gc := buildClient(gi.conf[keyToken])
576
577	ctx, cancel := context.WithTimeout(gi.iterator.ctx, defaultTimeout)
578	defer cancel()
579
580	err = gc.Query(ctx, &q, variables)
581	if err != nil {
582		return nil, err
583	}
584
585	var name string
586	if q.User.Name != nil {
587		name = string(*q.User.Name)
588	}
589
590	return repo.NewIdentityRaw(
591		name,
592		"",
593		string(q.User.Login),
594		string(q.User.AvatarUrl),
595		map[string]string{
596			keyGithubLogin: string(q.User.Login),
597		},
598	)
599}
600
601// parseId convert the unusable githubv4.ID (an interface{}) into a string
602func parseId(id githubv4.ID) string {
603	return fmt.Sprintf("%v", id)
604}