1package github
  2
  3import (
  4	"context"
  5	"fmt"
  6	"time"
  7
  8	"github.com/shurcooL/githubv4"
  9
 10	"github.com/MichaelMure/git-bug/bridge/core"
 11	"github.com/MichaelMure/git-bug/bridge/core/auth"
 12	"github.com/MichaelMure/git-bug/bug"
 13	"github.com/MichaelMure/git-bug/cache"
 14	"github.com/MichaelMure/git-bug/entity"
 15	"github.com/MichaelMure/git-bug/util/text"
 16)
 17
 18const EmptyTitlePlaceholder = "<empty string>"
 19
 20// githubImporter implement the Importer interface
 21type githubImporter struct {
 22	conf core.Configuration
 23
 24	// default client
 25	client *rateLimitHandlerClient
 26
 27	// mediator to access the Github API
 28	mediator *importMediator
 29
 30	// send only channel
 31	out chan<- core.ImportResult
 32}
 33
 34func (gi *githubImporter) Init(_ context.Context, repo *cache.RepoCache, conf core.Configuration) error {
 35	gi.conf = conf
 36	creds, err := auth.List(repo,
 37		auth.WithTarget(target),
 38		auth.WithKind(auth.KindToken),
 39		auth.WithMeta(auth.MetaKeyLogin, conf[confKeyDefaultLogin]),
 40	)
 41	if err != nil {
 42		return err
 43	}
 44	if len(creds) <= 0 {
 45		return ErrMissingIdentityToken
 46	}
 47	gi.client = buildClient(creds[0].(*auth.Token))
 48
 49	return nil
 50}
 51
 52// ImportAll iterate over all the configured repository issues and ensure the creation of the
 53// missing issues / timeline items / edits / label events ...
 54func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
 55	gi.mediator = NewImportMediator(ctx, gi.client, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
 56	out := make(chan core.ImportResult)
 57	gi.out = out
 58
 59	go func() {
 60		defer close(gi.out)
 61		var currBug *cache.BugCache
 62		var currEvent ImportEvent
 63		var nextEvent ImportEvent
 64		var err error
 65		for {
 66			// An IssueEvent contains the issue in its most recent state. If an issue
 67			// has at least one issue edit, then the history of the issue edits is
 68			// represented by IssueEditEvents. That is, the unedited (original) issue
 69			// might be saved only in the IssueEditEvent following the IssueEvent.
 70			// Since we replicate the edit history we need to either use the IssueEvent
 71			// (if there are no edits) or the IssueEvent together with its first
 72			// IssueEditEvent (if there are edits).
 73			// Exactly the same is true for comments and comment edits.
 74			// As a consequence we need to look at the current event and one look ahead
 75			// event.
 76
 77			currEvent = nextEvent
 78			if currEvent == nil {
 79				currEvent = gi.getEventHandleMsgs()
 80			}
 81			if currEvent == nil {
 82				break
 83			}
 84			nextEvent = gi.getEventHandleMsgs()
 85
 86			switch event := currEvent.(type) {
 87			case RateLimitingEvent:
 88				out <- core.NewImportRateLimiting(event.msg)
 89			case IssueEvent:
 90				// first: commit what is being held in currBug
 91				if err = gi.commit(currBug, out); err != nil {
 92					out <- core.NewImportError(err, "")
 93					return
 94				}
 95				// second: create new issue
 96				switch next := nextEvent.(type) {
 97				case IssueEditEvent:
 98					// consuming and using next event
 99					nextEvent = nil
100					currBug, err = gi.ensureIssue(ctx, repo, &event.issue, &next.userContentEdit)
101				default:
102					currBug, err = gi.ensureIssue(ctx, repo, &event.issue, nil)
103				}
104				if err != nil {
105					err := fmt.Errorf("issue creation: %v", err)
106					out <- core.NewImportError(err, "")
107					return
108				}
109			case IssueEditEvent:
110				err = gi.ensureIssueEdit(ctx, repo, currBug, event.issueId, &event.userContentEdit)
111				if err != nil {
112					err = fmt.Errorf("issue edit: %v", err)
113					out <- core.NewImportError(err, "")
114					return
115				}
116			case TimelineEvent:
117				if next, ok := nextEvent.(CommentEditEvent); ok && event.Typename == "IssueComment" {
118					// consuming and using next event
119					nextEvent = nil
120					err = gi.ensureComment(ctx, repo, currBug, &event.timelineItem.IssueComment, &next.userContentEdit)
121				} else {
122					err = gi.ensureTimelineItem(ctx, repo, currBug, &event.timelineItem)
123				}
124				if err != nil {
125					err = fmt.Errorf("timeline item creation: %v", err)
126					out <- core.NewImportError(err, "")
127					return
128				}
129			case CommentEditEvent:
130				err = gi.ensureCommentEdit(ctx, repo, currBug, event.commentId, &event.userContentEdit)
131				if err != nil {
132					err = fmt.Errorf("comment edit: %v", err)
133					out <- core.NewImportError(err, "")
134					return
135				}
136			default:
137				panic("Unknown event type")
138			}
139		}
140		// commit what is being held in currBug before returning
141		if err = gi.commit(currBug, out); err != nil {
142			out <- core.NewImportError(err, "")
143		}
144		if err = gi.mediator.Error(); err != nil {
145			gi.out <- core.NewImportError(err, "")
146		}
147	}()
148
149	return out, nil
150}
151
152func (gi *githubImporter) getEventHandleMsgs() ImportEvent {
153	for {
154		// read event from import mediator
155		event := gi.mediator.NextImportEvent()
156		// consume (and use) all rate limiting events
157		if e, ok := event.(RateLimitingEvent); ok {
158			gi.out <- core.NewImportRateLimiting(e.msg)
159			continue
160		}
161		return event
162	}
163}
164
165func (gi *githubImporter) commit(b *cache.BugCache, out chan<- core.ImportResult) error {
166	if b == nil {
167		return nil
168	}
169	if !b.NeedCommit() {
170		out <- core.NewImportNothing(b.Id(), "no imported operation")
171		return nil
172	} else if err := b.Commit(); err != nil {
173		// commit bug state
174		return fmt.Errorf("bug commit: %v", err)
175	}
176	return nil
177}
178
179func (gi *githubImporter) ensureIssue(ctx context.Context, repo *cache.RepoCache, issue *issue, issueEdit *userContentEdit) (*cache.BugCache, error) {
180	author, err := gi.ensurePerson(ctx, repo, issue.Author)
181	if err != nil {
182		return nil, err
183	}
184
185	// resolve bug
186	b, err := repo.ResolveBugMatcher(func(excerpt *cache.BugExcerpt) bool {
187		return excerpt.CreateMetadata[metaKeyGithubUrl] == issue.Url.String() &&
188			excerpt.CreateMetadata[metaKeyGithubId] == parseId(issue.Id)
189	})
190	if err == nil {
191		return b, nil
192	}
193	if err != bug.ErrBugNotExist {
194		return nil, err
195	}
196
197	// At Github there exist issues with seemingly empty titles. An example is
198	// https://github.com/NixOS/nixpkgs/issues/72730 .
199	// The title provided by the GraphQL API actually consists of a space followed by a
200	// zero width space (U+200B). This title would cause the NewBugRaw() function to
201	// return an error: empty title.
202	title := string(issue.Title)
203	if title == " \u200b" { // U+200B == zero width space
204		title = EmptyTitlePlaceholder
205	}
206
207	var textInput string
208	if issueEdit != nil {
209		// use the first issue edit: it represents the bug creation itself
210		textInput = string(*issueEdit.Diff)
211	} else {
212		// if there are no issue edits then the issue struct holds the bug creation
213		textInput = string(issue.Body)
214	}
215
216	// create bug
217	b, _, err = repo.NewBugRaw(
218		author,
219		issue.CreatedAt.Unix(),
220		text.CleanupOneLine(title), // TODO: this is the *current* title, not the original one
221		text.Cleanup(textInput),
222		nil,
223		map[string]string{
224			core.MetaKeyOrigin: target,
225			metaKeyGithubId:    parseId(issue.Id),
226			metaKeyGithubUrl:   issue.Url.String(),
227		})
228	if err != nil {
229		return nil, err
230	}
231	// importing a new bug
232	gi.out <- core.NewImportBug(b.Id())
233
234	return b, nil
235}
236
237func (gi *githubImporter) ensureIssueEdit(ctx context.Context, repo *cache.RepoCache, bug *cache.BugCache, ghIssueId githubv4.ID, edit *userContentEdit) error {
238	return gi.ensureCommentEdit(ctx, repo, bug, ghIssueId, edit)
239}
240
241func (gi *githubImporter) ensureTimelineItem(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, item *timelineItem) error {
242
243	switch item.Typename {
244	case "IssueComment":
245		err := gi.ensureComment(ctx, repo, b, &item.IssueComment, nil)
246		if err != nil {
247			return fmt.Errorf("timeline comment creation: %v", err)
248		}
249		return nil
250
251	case "LabeledEvent":
252		id := parseId(item.LabeledEvent.Id)
253		_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
254		if err == nil {
255			return nil
256		}
257
258		if err != cache.ErrNoMatchingOp {
259			return err
260		}
261		author, err := gi.ensurePerson(ctx, repo, item.LabeledEvent.Actor)
262		if err != nil {
263			return err
264		}
265		op, err := b.ForceChangeLabelsRaw(
266			author,
267			item.LabeledEvent.CreatedAt.Unix(),
268			[]string{
269				text.CleanupOneLine(string(item.LabeledEvent.Label.Name)),
270			},
271			nil,
272			map[string]string{metaKeyGithubId: id},
273		)
274		if err != nil {
275			return err
276		}
277
278		gi.out <- core.NewImportLabelChange(op.Id())
279		return nil
280
281	case "UnlabeledEvent":
282		id := parseId(item.UnlabeledEvent.Id)
283		_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
284		if err == nil {
285			return nil
286		}
287		if err != cache.ErrNoMatchingOp {
288			return err
289		}
290		author, err := gi.ensurePerson(ctx, repo, item.UnlabeledEvent.Actor)
291		if err != nil {
292			return err
293		}
294
295		op, err := b.ForceChangeLabelsRaw(
296			author,
297			item.UnlabeledEvent.CreatedAt.Unix(),
298			nil,
299			[]string{
300				text.CleanupOneLine(string(item.UnlabeledEvent.Label.Name)),
301			},
302			map[string]string{metaKeyGithubId: id},
303		)
304		if err != nil {
305			return err
306		}
307
308		gi.out <- core.NewImportLabelChange(op.Id())
309		return nil
310
311	case "ClosedEvent":
312		id := parseId(item.ClosedEvent.Id)
313		_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
314		if err != cache.ErrNoMatchingOp {
315			return err
316		}
317		if err == nil {
318			return nil
319		}
320		author, err := gi.ensurePerson(ctx, repo, item.ClosedEvent.Actor)
321		if err != nil {
322			return err
323		}
324		op, err := b.CloseRaw(
325			author,
326			item.ClosedEvent.CreatedAt.Unix(),
327			map[string]string{metaKeyGithubId: id},
328		)
329
330		if err != nil {
331			return err
332		}
333
334		gi.out <- core.NewImportStatusChange(op.Id())
335		return nil
336
337	case "ReopenedEvent":
338		id := parseId(item.ReopenedEvent.Id)
339		_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
340		if err != cache.ErrNoMatchingOp {
341			return err
342		}
343		if err == nil {
344			return nil
345		}
346		author, err := gi.ensurePerson(ctx, repo, item.ReopenedEvent.Actor)
347		if err != nil {
348			return err
349		}
350		op, err := b.OpenRaw(
351			author,
352			item.ReopenedEvent.CreatedAt.Unix(),
353			map[string]string{metaKeyGithubId: id},
354		)
355
356		if err != nil {
357			return err
358		}
359
360		gi.out <- core.NewImportStatusChange(op.Id())
361		return nil
362
363	case "RenamedTitleEvent":
364		id := parseId(item.RenamedTitleEvent.Id)
365		_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
366		if err != cache.ErrNoMatchingOp {
367			return err
368		}
369		if err == nil {
370			return nil
371		}
372		author, err := gi.ensurePerson(ctx, repo, item.RenamedTitleEvent.Actor)
373		if err != nil {
374			return err
375		}
376
377		// At Github there exist issues with seemingly empty titles. An example is
378		// https://github.com/NixOS/nixpkgs/issues/72730 .
379		// The title provided by the GraphQL API actually consists of a space followed
380		// by a zero width space (U+200B). This title would cause the NewBugRaw()
381		// function to return an error: empty title.
382		title := text.CleanupOneLine(string(item.RenamedTitleEvent.CurrentTitle))
383		if title == " \u200b" { // U+200B == zero width space
384			title = EmptyTitlePlaceholder
385		}
386
387		op, err := b.SetTitleRaw(
388			author,
389			item.RenamedTitleEvent.CreatedAt.Unix(),
390			title,
391			map[string]string{metaKeyGithubId: id},
392		)
393		if err != nil {
394			return err
395		}
396
397		gi.out <- core.NewImportTitleEdition(op.Id())
398		return nil
399	}
400
401	return nil
402}
403
404func (gi *githubImporter) ensureCommentEdit(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, ghTargetId githubv4.ID, edit *userContentEdit) error {
405	// find comment
406	target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(ghTargetId))
407	if err != nil {
408		return err
409	}
410	_, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
411	if err == nil {
412		return nil
413	}
414	if err != cache.ErrNoMatchingOp {
415		// real error
416		return err
417	}
418
419	editor, err := gi.ensurePerson(ctx, repo, edit.Editor)
420	if err != nil {
421		return err
422	}
423
424	if edit.DeletedAt != nil {
425		// comment deletion, not supported yet
426		return nil
427	}
428
429	// comment edition
430	op, err := b.EditCommentRaw(
431		editor,
432		edit.CreatedAt.Unix(),
433		target,
434		text.Cleanup(string(*edit.Diff)),
435		map[string]string{
436			metaKeyGithubId: parseId(edit.Id),
437		},
438	)
439
440	if err != nil {
441		return err
442	}
443
444	gi.out <- core.NewImportCommentEdition(op.Id())
445	return nil
446}
447
448func (gi *githubImporter) ensureComment(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, comment *issueComment, firstEdit *userContentEdit) error {
449	author, err := gi.ensurePerson(ctx, repo, comment.Author)
450	if err != nil {
451		return err
452	}
453
454	_, err = b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(comment.Id))
455	if err == nil {
456		return nil
457	}
458	if err != cache.ErrNoMatchingOp {
459		// real error
460		return err
461	}
462
463	var textInput string
464	if firstEdit != nil {
465		// use the first comment edit: it represents the comment creation itself
466		textInput = string(*firstEdit.Diff)
467	} else {
468		// if there are not comment edits, then the comment struct holds the comment creation
469		textInput = string(comment.Body)
470	}
471
472	// add comment operation
473	op, err := b.AddCommentRaw(
474		author,
475		comment.CreatedAt.Unix(),
476		text.Cleanup(textInput),
477		nil,
478		map[string]string{
479			metaKeyGithubId:  parseId(comment.Id),
480			metaKeyGithubUrl: comment.Url.String(),
481		},
482	)
483	if err != nil {
484		return err
485	}
486
487	gi.out <- core.NewImportComment(op.Id())
488	return nil
489}
490
491// ensurePerson create a bug.Person from the Github data
492func (gi *githubImporter) ensurePerson(ctx context.Context, repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
493	// When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
494	// in it's UI. So we need a special case to get it.
495	if actor == nil {
496		return gi.getGhost(ctx, repo)
497	}
498
499	// Look first in the cache
500	i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
501	if err == nil {
502		return i, nil
503	}
504	if entity.IsErrMultipleMatch(err) {
505		return nil, err
506	}
507
508	// importing a new identity
509	var name string
510	var email string
511
512	switch actor.Typename {
513	case "User":
514		if actor.User.Name != nil {
515			name = string(*(actor.User.Name))
516		}
517		email = string(actor.User.Email)
518	case "Organization":
519		if actor.Organization.Name != nil {
520			name = string(*(actor.Organization.Name))
521		}
522		if actor.Organization.Email != nil {
523			email = string(*(actor.Organization.Email))
524		}
525	case "Bot":
526	}
527
528	// Name is not necessarily set, fallback to login as a name is required in the identity
529	if name == "" {
530		name = string(actor.Login)
531	}
532
533	i, err = repo.NewIdentityRaw(
534		name,
535		email,
536		string(actor.Login),
537		string(actor.AvatarUrl),
538		nil,
539		map[string]string{
540			metaKeyGithubLogin: string(actor.Login),
541		},
542	)
543
544	if err != nil {
545		return nil, err
546	}
547
548	gi.out <- core.NewImportIdentity(i.Id())
549	return i, nil
550}
551
552func (gi *githubImporter) getGhost(ctx context.Context, repo *cache.RepoCache) (*cache.IdentityCache, error) {
553	loginName := "ghost"
554	// Look first in the cache
555	i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, loginName)
556	if err == nil {
557		return i, nil
558	}
559	if entity.IsErrMultipleMatch(err) {
560		return nil, err
561	}
562	user, err := gi.mediator.User(ctx, loginName)
563	if err != nil {
564		return nil, err
565	}
566	userName := ""
567	if user.Name != nil {
568		userName = string(*user.Name)
569	}
570	return repo.NewIdentityRaw(
571		userName,
572		"",
573		string(user.Login),
574		string(user.AvatarUrl),
575		nil,
576		map[string]string{
577			metaKeyGithubLogin: string(user.Login),
578		},
579	)
580}
581
582// parseId converts the unusable githubv4.ID (an interface{}) into a string
583func parseId(id githubv4.ID) string {
584	return fmt.Sprintf("%v", id)
585}