1package github
  2
  3import (
  4	"context"
  5	"fmt"
  6	"time"
  7
  8	"github.com/shurcooL/githubv4"
  9
 10	"github.com/MichaelMure/git-bug/bridge/core"
 11	"github.com/MichaelMure/git-bug/bug"
 12	"github.com/MichaelMure/git-bug/cache"
 13	"github.com/MichaelMure/git-bug/entity"
 14	"github.com/MichaelMure/git-bug/util/text"
 15)
 16
 17const (
 18	keyOrigin      = "origin"
 19	keyGithubId    = "github-id"
 20	keyGithubUrl   = "github-url"
 21	keyGithubLogin = "github-login"
 22)
 23
 24// githubImporter implement the Importer interface
 25type githubImporter struct {
 26	conf core.Configuration
 27
 28	// iterator
 29	iterator *iterator
 30
 31	// send only channel
 32	out chan<- core.ImportResult
 33}
 34
 35func (gi *githubImporter) Init(conf core.Configuration) error {
 36	gi.conf = conf
 37	return nil
 38}
 39
 40// ImportAll iterate over all the configured repository issues and ensure the creation of the
 41// missing issues / timeline items / edits / label events ...
 42func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
 43	gi.iterator = NewIterator(ctx, 10, gi.conf[keyOwner], gi.conf[keyProject], gi.conf[keyToken], since)
 44	out := make(chan core.ImportResult)
 45	gi.out = out
 46
 47	go func() {
 48		defer close(gi.out)
 49
 50		// Loop over all matching issues
 51		for gi.iterator.NextIssue() {
 52			issue := gi.iterator.IssueValue()
 53			// create issue
 54			b, err := gi.ensureIssue(repo, issue)
 55			if err != nil {
 56				err := fmt.Errorf("issue creation: %v", err)
 57				out <- core.NewImportError(err, "")
 58				return
 59			}
 60
 61			// loop over timeline items
 62			for gi.iterator.NextTimelineItem() {
 63				item := gi.iterator.TimelineItemValue()
 64				if err := gi.ensureTimelineItem(repo, b, item); err != nil {
 65					err := fmt.Errorf("timeline item creation: %v", err)
 66					out <- core.NewImportError(err, "")
 67					return
 68				}
 69			}
 70
 71			// commit bug state
 72			if err := b.CommitAsNeeded(); err != nil {
 73				err = fmt.Errorf("bug commit: %v", err)
 74				out <- core.NewImportError(err, "")
 75				return
 76			}
 77		}
 78
 79		if err := gi.iterator.Error(); err != nil && err != context.Canceled {
 80			gi.out <- core.NewImportError(err, "")
 81		}
 82	}()
 83
 84	return out, nil
 85}
 86
 87func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline) (*cache.BugCache, error) {
 88	// ensure issue author
 89	author, err := gi.ensurePerson(repo, issue.Author)
 90	if err != nil {
 91		return nil, err
 92	}
 93
 94	// resolve bug
 95	b, err := repo.ResolveBugCreateMetadata(keyGithubUrl, issue.Url.String())
 96	if err != nil && err != bug.ErrBugNotExist {
 97		return nil, err
 98	}
 99
100	// get issue edits
101	var issueEdits []userContentEdit
102	for gi.iterator.NextIssueEdit() {
103		issueEdits = append(issueEdits, gi.iterator.IssueEditValue())
104	}
105
106	// if issueEdits is empty
107	if len(issueEdits) == 0 {
108		if err == bug.ErrBugNotExist {
109			cleanText, err := text.Cleanup(string(issue.Body))
110			if err != nil {
111				return nil, err
112			}
113
114			// create bug
115			b, _, err = repo.NewBugRaw(
116				author,
117				issue.CreatedAt.Unix(),
118				issue.Title,
119				cleanText,
120				nil,
121				map[string]string{
122					keyOrigin:    target,
123					keyGithubId:  parseId(issue.Id),
124					keyGithubUrl: issue.Url.String(),
125				})
126			if err != nil {
127				return nil, err
128			}
129
130			// importing a new bug
131			gi.out <- core.NewImportBug(b.Id())
132		} else {
133			gi.out <- core.NewImportNothing("", "bug already imported")
134		}
135
136	} else {
137		// create bug from given issueEdits
138		for i, edit := range issueEdits {
139			if i == 0 && b != nil {
140				// The first edit in the github result is the issue creation itself, we already have that
141				gi.out <- core.NewImportNothing("", "bug already imported")
142				continue
143			}
144
145			cleanText, err := text.Cleanup(string(*edit.Diff))
146			if err != nil {
147				return nil, err
148			}
149
150			// if the bug doesn't exist
151			if b == nil {
152				// we create the bug as soon as we have a legit first edition
153				b, _, err = repo.NewBugRaw(
154					author,
155					issue.CreatedAt.Unix(),
156					issue.Title,
157					cleanText,
158					nil,
159					map[string]string{
160						keyOrigin:    target,
161						keyGithubId:  parseId(issue.Id),
162						keyGithubUrl: issue.Url.String(),
163					},
164				)
165
166				if err != nil {
167					return nil, err
168				}
169
170				// importing a new bug
171				gi.out <- core.NewImportBug(b.Id())
172				continue
173			}
174
175			// other edits will be added as CommentEdit operations
176			target, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(issue.Id))
177			if err != nil {
178				return nil, err
179			}
180
181			err = gi.ensureCommentEdit(repo, b, target, edit)
182			if err != nil {
183				return nil, err
184			}
185		}
186	}
187
188	return b, nil
189}
190
191func (gi *githubImporter) ensureTimelineItem(repo *cache.RepoCache, b *cache.BugCache, item timelineItem) error {
192
193	switch item.Typename {
194	case "IssueComment":
195		// collect all comment edits
196		var commentEdits []userContentEdit
197		for gi.iterator.NextCommentEdit() {
198			commentEdits = append(commentEdits, gi.iterator.CommentEditValue())
199		}
200
201		// ensureTimelineComment send import events over out chanel
202		err := gi.ensureTimelineComment(repo, b, item.IssueComment, commentEdits)
203		if err != nil {
204			return fmt.Errorf("timeline comment creation: %v", err)
205		}
206
207	case "LabeledEvent":
208		id := parseId(item.LabeledEvent.Id)
209		_, err := b.ResolveOperationWithMetadata(keyGithubId, id)
210		if err == nil {
211			reason := fmt.Sprintf("operation already imported: %v", item.Typename)
212			gi.out <- core.NewImportNothing("", reason)
213			return nil
214		}
215
216		if err != cache.ErrNoMatchingOp {
217			return err
218		}
219		author, err := gi.ensurePerson(repo, item.LabeledEvent.Actor)
220		if err != nil {
221			return err
222		}
223		op, err := b.ForceChangeLabelsRaw(
224			author,
225			item.LabeledEvent.CreatedAt.Unix(),
226			[]string{
227				string(item.LabeledEvent.Label.Name),
228			},
229			nil,
230			map[string]string{keyGithubId: id},
231		)
232		if err != nil {
233			return err
234		}
235
236		gi.out <- core.NewImportLabelChange(op.Id())
237		return nil
238
239	case "UnlabeledEvent":
240		id := parseId(item.UnlabeledEvent.Id)
241		_, err := b.ResolveOperationWithMetadata(keyGithubId, id)
242		if err == nil {
243			reason := fmt.Sprintf("operation already imported: %v", item.Typename)
244			gi.out <- core.NewImportNothing("", reason)
245			return nil
246		}
247		if err != cache.ErrNoMatchingOp {
248			return err
249		}
250		author, err := gi.ensurePerson(repo, item.UnlabeledEvent.Actor)
251		if err != nil {
252			return err
253		}
254
255		op, err := b.ForceChangeLabelsRaw(
256			author,
257			item.UnlabeledEvent.CreatedAt.Unix(),
258			nil,
259			[]string{
260				string(item.UnlabeledEvent.Label.Name),
261			},
262			map[string]string{keyGithubId: id},
263		)
264		if err != nil {
265			return err
266		}
267
268		gi.out <- core.NewImportLabelChange(op.Id())
269		return nil
270
271	case "ClosedEvent":
272		id := parseId(item.ClosedEvent.Id)
273		_, err := b.ResolveOperationWithMetadata(keyGithubId, id)
274		if err != cache.ErrNoMatchingOp {
275			return err
276		}
277		if err == nil {
278			reason := fmt.Sprintf("operation already imported: %v", item.Typename)
279			gi.out <- core.NewImportNothing("", reason)
280			return nil
281		}
282		author, err := gi.ensurePerson(repo, item.ClosedEvent.Actor)
283		if err != nil {
284			return err
285		}
286		op, err := b.CloseRaw(
287			author,
288			item.ClosedEvent.CreatedAt.Unix(),
289			map[string]string{keyGithubId: id},
290		)
291
292		if err != nil {
293			return err
294		}
295
296		gi.out <- core.NewImportStatusChange(op.Id())
297		return nil
298
299	case "ReopenedEvent":
300		id := parseId(item.ReopenedEvent.Id)
301		_, err := b.ResolveOperationWithMetadata(keyGithubId, id)
302		if err != cache.ErrNoMatchingOp {
303			return err
304		}
305		if err == nil {
306			reason := fmt.Sprintf("operation already imported: %v", item.Typename)
307			gi.out <- core.NewImportNothing("", reason)
308			return nil
309		}
310		author, err := gi.ensurePerson(repo, item.ReopenedEvent.Actor)
311		if err != nil {
312			return err
313		}
314		op, err := b.OpenRaw(
315			author,
316			item.ReopenedEvent.CreatedAt.Unix(),
317			map[string]string{keyGithubId: id},
318		)
319
320		if err != nil {
321			return err
322		}
323
324		gi.out <- core.NewImportStatusChange(op.Id())
325		return nil
326
327	case "RenamedTitleEvent":
328		id := parseId(item.RenamedTitleEvent.Id)
329		_, err := b.ResolveOperationWithMetadata(keyGithubId, id)
330		if err != cache.ErrNoMatchingOp {
331			return err
332		}
333		if err == nil {
334			reason := fmt.Sprintf("operation already imported: %v", item.Typename)
335			gi.out <- core.NewImportNothing("", reason)
336			return nil
337		}
338		author, err := gi.ensurePerson(repo, item.RenamedTitleEvent.Actor)
339		if err != nil {
340			return err
341		}
342		op, err := b.SetTitleRaw(
343			author,
344			item.RenamedTitleEvent.CreatedAt.Unix(),
345			string(item.RenamedTitleEvent.CurrentTitle),
346			map[string]string{keyGithubId: id},
347		)
348		if err != nil {
349			return err
350		}
351
352		gi.out <- core.NewImportTitleEdition(op.Id())
353		return nil
354
355	default:
356		reason := fmt.Sprintf("ignoring timeline type: %v", item.Typename)
357		gi.out <- core.NewImportNothing("", reason)
358	}
359
360	return nil
361}
362
363func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache.BugCache, item issueComment, edits []userContentEdit) error {
364	// ensure person
365	author, err := gi.ensurePerson(repo, item.Author)
366	if err != nil {
367		return err
368	}
369
370	targetOpID, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(item.Id))
371	if err == nil {
372		gi.out <- core.NewImportNothing("", "comment already imported")
373	} else if err != cache.ErrNoMatchingOp {
374		// real error
375		return err
376	}
377
378	// if no edits are given we create the comment
379	if len(edits) == 0 {
380		if err == cache.ErrNoMatchingOp {
381			cleanText, err := text.Cleanup(string(item.Body))
382			if err != nil {
383				return err
384			}
385
386			// add comment operation
387			op, err := b.AddCommentRaw(
388				author,
389				item.CreatedAt.Unix(),
390				cleanText,
391				nil,
392				map[string]string{
393					keyGithubId:  parseId(item.Id),
394					keyGithubUrl: parseId(item.Url.String()),
395				},
396			)
397			if err != nil {
398				return err
399			}
400
401			gi.out <- core.NewImportComment(op.Id())
402		}
403
404	} else {
405		for i, edit := range edits {
406			if i == 0 && targetOpID != "" {
407				// The first edit in the github result is the comment creation itself, we already have that
408				gi.out <- core.NewImportNothing("", "comment already imported")
409				continue
410			}
411
412			// ensure editor identity
413			editor, err := gi.ensurePerson(repo, edit.Editor)
414			if err != nil {
415				return err
416			}
417
418			// create comment when target is empty
419			if targetOpID == "" {
420				cleanText, err := text.Cleanup(string(*edit.Diff))
421				if err != nil {
422					return err
423				}
424
425				op, err := b.AddCommentRaw(
426					editor,
427					edit.CreatedAt.Unix(),
428					cleanText,
429					nil,
430					map[string]string{
431						keyGithubId:  parseId(item.Id),
432						keyGithubUrl: item.Url.String(),
433					},
434				)
435				if err != nil {
436					return err
437				}
438
439				// set target for the nexr edit now that the comment is created
440				targetOpID = op.Id()
441				continue
442			}
443
444			err = gi.ensureCommentEdit(repo, b, targetOpID, edit)
445			if err != nil {
446				return err
447			}
448		}
449	}
450	return nil
451}
452
453func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugCache, target entity.Id, edit userContentEdit) error {
454	_, err := b.ResolveOperationWithMetadata(keyGithubId, parseId(edit.Id))
455	if err == nil {
456		gi.out <- core.NewImportNothing(b.Id(), "edition already imported")
457		return nil
458	}
459	if err != cache.ErrNoMatchingOp {
460		// real error
461		return err
462	}
463
464	editor, err := gi.ensurePerson(repo, edit.Editor)
465	if err != nil {
466		return err
467	}
468
469	switch {
470	case edit.DeletedAt != nil:
471		// comment deletion, not supported yet
472		gi.out <- core.NewImportNothing(b.Id(), "comment deletion is not supported yet")
473
474	case edit.DeletedAt == nil:
475
476		cleanText, err := text.Cleanup(string(*edit.Diff))
477		if err != nil {
478			return err
479		}
480
481		// comment edition
482		op, err := b.EditCommentRaw(
483			editor,
484			edit.CreatedAt.Unix(),
485			target,
486			cleanText,
487			map[string]string{
488				keyGithubId: parseId(edit.Id),
489			},
490		)
491
492		if err != nil {
493			return err
494		}
495
496		gi.out <- core.NewImportCommentEdition(op.Id())
497	}
498
499	return nil
500}
501
502// ensurePerson create a bug.Person from the Github data
503func (gi *githubImporter) ensurePerson(repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
504	// When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
505	// in it's UI. So we need a special case to get it.
506	if actor == nil {
507		return gi.getGhost(repo)
508	}
509
510	// Look first in the cache
511	i, err := repo.ResolveIdentityImmutableMetadata(keyGithubLogin, string(actor.Login))
512	if err == nil {
513		return i, nil
514	}
515	if _, ok := err.(entity.ErrMultipleMatch); ok {
516		return nil, err
517	}
518
519	// importing a new identity
520
521	var name string
522	var email string
523
524	switch actor.Typename {
525	case "User":
526		if actor.User.Name != nil {
527			name = string(*(actor.User.Name))
528		}
529		email = string(actor.User.Email)
530	case "Organization":
531		if actor.Organization.Name != nil {
532			name = string(*(actor.Organization.Name))
533		}
534		if actor.Organization.Email != nil {
535			email = string(*(actor.Organization.Email))
536		}
537	case "Bot":
538	}
539
540	i, err = repo.NewIdentityRaw(
541		name,
542		email,
543		string(actor.Login),
544		string(actor.AvatarUrl),
545		map[string]string{
546			keyGithubLogin: string(actor.Login),
547		},
548	)
549
550	if err != nil {
551		return nil, err
552	}
553
554	gi.out <- core.NewImportIdentity(i.Id())
555	return i, nil
556}
557
558func (gi *githubImporter) getGhost(repo *cache.RepoCache) (*cache.IdentityCache, error) {
559	// Look first in the cache
560	i, err := repo.ResolveIdentityImmutableMetadata(keyGithubLogin, "ghost")
561	if err == nil {
562		return i, nil
563	}
564	if _, ok := err.(entity.ErrMultipleMatch); ok {
565		return nil, err
566	}
567
568	var q ghostQuery
569
570	variables := map[string]interface{}{
571		"login": githubv4.String("ghost"),
572	}
573
574	gc := buildClient(gi.conf[keyToken])
575
576	ctx, cancel := context.WithTimeout(gi.iterator.ctx, defaultTimeout)
577	defer cancel()
578
579	err = gc.Query(ctx, &q, variables)
580	if err != nil {
581		return nil, err
582	}
583
584	var name string
585	if q.User.Name != nil {
586		name = string(*q.User.Name)
587	}
588
589	return repo.NewIdentityRaw(
590		name,
591		"",
592		string(q.User.Login),
593		string(q.User.AvatarUrl),
594		map[string]string{
595			keyGithubLogin: string(q.User.Login),
596		},
597	)
598}
599
600// parseId convert the unusable githubv4.ID (an interface{}) into a string
601func parseId(id githubv4.ID) string {
602	return fmt.Sprintf("%v", id)
603}