1package github
  2
  3import (
  4	"context"
  5	"fmt"
  6	"time"
  7
  8	"github.com/shurcooL/githubv4"
  9
 10	"github.com/MichaelMure/git-bug/bridge/core"
 11	"github.com/MichaelMure/git-bug/bridge/core/auth"
 12	"github.com/MichaelMure/git-bug/bug"
 13	"github.com/MichaelMure/git-bug/cache"
 14	"github.com/MichaelMure/git-bug/entity"
 15	"github.com/MichaelMure/git-bug/util/text"
 16)
 17
 18const EMPTY_TITLE_PLACEHOLDER = "<empty string>"
 19
 20// githubImporter implement the Importer interface
 21type githubImporter struct {
 22	conf core.Configuration
 23
 24	// mediator to access the Github API
 25	mediator *importMediator
 26
 27	// send only channel
 28	out chan<- core.ImportResult
 29}
 30
 31func (gi *githubImporter) Init(_ context.Context, _ *cache.RepoCache, conf core.Configuration) error {
 32	gi.conf = conf
 33	return nil
 34}
 35
 36// ImportAll iterate over all the configured repository issues and ensure the creation of the
 37// missing issues / timeline items / edits / label events ...
 38func (gi *githubImporter) ImportAll(ctx context.Context, repo *cache.RepoCache, since time.Time) (<-chan core.ImportResult, error) {
 39	creds, err := auth.List(repo,
 40		auth.WithTarget(target),
 41		auth.WithKind(auth.KindToken),
 42		auth.WithMeta(auth.MetaKeyLogin, gi.conf[confKeyDefaultLogin]),
 43	)
 44	if err != nil {
 45		return nil, err
 46	}
 47	if len(creds) <= 0 {
 48		return nil, ErrMissingIdentityToken
 49	}
 50	client := buildClient(creds[0].(*auth.Token))
 51	gi.mediator = NewImportMediator(ctx, client, gi.conf[confKeyOwner], gi.conf[confKeyProject], since)
 52	out := make(chan core.ImportResult)
 53	gi.out = out
 54
 55	go func() {
 56		defer close(gi.out)
 57
 58		// Loop over all matching issues
 59		for issue := range gi.mediator.Issues() {
 60			// create issue
 61			b, err := gi.ensureIssue(ctx, repo, &issue)
 62			if err != nil {
 63				err := fmt.Errorf("issue creation: %v", err)
 64				out <- core.NewImportError(err, "")
 65				return
 66			}
 67
 68			// loop over timeline items
 69			for item := range gi.mediator.TimelineItems(&issue) {
 70				err := gi.ensureTimelineItem(ctx, repo, b, item)
 71				if err != nil {
 72					err = fmt.Errorf("timeline item creation: %v", err)
 73					out <- core.NewImportError(err, "")
 74					return
 75				}
 76			}
 77
 78			if !b.NeedCommit() {
 79				out <- core.NewImportNothing(b.Id(), "no imported operation")
 80			} else if err := b.Commit(); err != nil {
 81				// commit bug state
 82				err = fmt.Errorf("bug commit: %v", err)
 83				out <- core.NewImportError(err, "")
 84				return
 85			}
 86		}
 87
 88		if err := gi.mediator.Error(); err != nil {
 89			gi.out <- core.NewImportError(err, "")
 90		}
 91	}()
 92
 93	return out, nil
 94}
 95
 96func (gi *githubImporter) ensureIssue(ctx context.Context, repo *cache.RepoCache, issue *issue) (*cache.BugCache, error) {
 97	author, err := gi.ensurePerson(ctx, repo, issue.Author)
 98	if err != nil {
 99		return nil, err
100	}
101
102	// resolve bug
103	b, err := repo.ResolveBugMatcher(func(excerpt *cache.BugExcerpt) bool {
104		return excerpt.CreateMetadata[core.MetaKeyOrigin] == target &&
105			excerpt.CreateMetadata[metaKeyGithubId] == parseId(issue.Id)
106	})
107	if err != nil && err != bug.ErrBugNotExist {
108		return nil, err
109	}
110
111	// get first issue edit
112	// if it exists, then it holds the bug creation
113	firstEdit, hasEdit := <-gi.mediator.IssueEdits(issue)
114
115	// At Github there exist issues with seemingly empty titles. An example is
116	// https://github.com/NixOS/nixpkgs/issues/72730 .
117	// The title provided by the GraphQL API actually consists of a space followed by a
118	// zero width space (U+200B). This title would cause the NewBugRaw() function to
119	// return an error: empty title.
120	title := string(issue.Title)
121	if title == " \u200b" { // U+200B == zero width space
122		title = EMPTY_TITLE_PLACEHOLDER
123	}
124
125	if err == bug.ErrBugNotExist {
126		var textInput string
127		if hasEdit {
128			// use the first issue edit: it represents the bug creation itself
129			textInput = string(*firstEdit.Diff)
130		} else {
131			// if there are no issue edits then the issue struct holds the bug creation
132			textInput = string(issue.Body)
133		}
134		cleanText, err := text.Cleanup(textInput)
135		if err != nil {
136			return nil, err
137		}
138		// create bug
139		b, _, err = repo.NewBugRaw(
140			author,
141			issue.CreatedAt.Unix(),
142			title, // TODO: this is the *current* title, not the original one
143			cleanText,
144			nil,
145			map[string]string{
146				core.MetaKeyOrigin: target,
147				metaKeyGithubId:    parseId(issue.Id),
148				metaKeyGithubUrl:   issue.Url.String(),
149			})
150		if err != nil {
151			return nil, err
152		}
153		// importing a new bug
154		gi.out <- core.NewImportBug(b.Id())
155	}
156	if b == nil {
157		return nil, fmt.Errorf("finding or creating issue")
158	}
159	// process remaining issue edits, if they exist
160	for edit := range gi.mediator.IssueEdits(issue) {
161		// other edits will be added as CommentEdit operations
162		target, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(issue.Id))
163		if err == cache.ErrNoMatchingOp {
164			// original comment is missing somehow, issuing a warning
165			gi.out <- core.NewImportWarning(fmt.Errorf("comment ID %s to edit is missing", parseId(issue.Id)), b.Id())
166			continue
167		}
168		if err != nil {
169			return nil, err
170		}
171
172		err = gi.ensureCommentEdit(ctx, repo, b, target, edit)
173		if err != nil {
174			return nil, err
175		}
176	}
177	return b, nil
178}
179
180func (gi *githubImporter) ensureTimelineItem(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, item timelineItem) error {
181
182	switch item.Typename {
183	case "IssueComment":
184		err := gi.ensureComment(ctx, repo, b, &item.IssueComment)
185		if err != nil {
186			return fmt.Errorf("timeline comment creation: %v", err)
187		}
188		return nil
189
190	case "LabeledEvent":
191		id := parseId(item.LabeledEvent.Id)
192		_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
193		if err == nil {
194			return nil
195		}
196
197		if err != cache.ErrNoMatchingOp {
198			return err
199		}
200		author, err := gi.ensurePerson(ctx, repo, item.LabeledEvent.Actor)
201		if err != nil {
202			return err
203		}
204		op, err := b.ForceChangeLabelsRaw(
205			author,
206			item.LabeledEvent.CreatedAt.Unix(),
207			[]string{
208				string(item.LabeledEvent.Label.Name),
209			},
210			nil,
211			map[string]string{metaKeyGithubId: id},
212		)
213		if err != nil {
214			return err
215		}
216
217		gi.out <- core.NewImportLabelChange(op.Id())
218		return nil
219
220	case "UnlabeledEvent":
221		id := parseId(item.UnlabeledEvent.Id)
222		_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
223		if err == nil {
224			return nil
225		}
226		if err != cache.ErrNoMatchingOp {
227			return err
228		}
229		author, err := gi.ensurePerson(ctx, repo, item.UnlabeledEvent.Actor)
230		if err != nil {
231			return err
232		}
233
234		op, err := b.ForceChangeLabelsRaw(
235			author,
236			item.UnlabeledEvent.CreatedAt.Unix(),
237			nil,
238			[]string{
239				string(item.UnlabeledEvent.Label.Name),
240			},
241			map[string]string{metaKeyGithubId: id},
242		)
243		if err != nil {
244			return err
245		}
246
247		gi.out <- core.NewImportLabelChange(op.Id())
248		return nil
249
250	case "ClosedEvent":
251		id := parseId(item.ClosedEvent.Id)
252		_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
253		if err != cache.ErrNoMatchingOp {
254			return err
255		}
256		if err == nil {
257			return nil
258		}
259		author, err := gi.ensurePerson(ctx, repo, item.ClosedEvent.Actor)
260		if err != nil {
261			return err
262		}
263		op, err := b.CloseRaw(
264			author,
265			item.ClosedEvent.CreatedAt.Unix(),
266			map[string]string{metaKeyGithubId: id},
267		)
268
269		if err != nil {
270			return err
271		}
272
273		gi.out <- core.NewImportStatusChange(op.Id())
274		return nil
275
276	case "ReopenedEvent":
277		id := parseId(item.ReopenedEvent.Id)
278		_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
279		if err != cache.ErrNoMatchingOp {
280			return err
281		}
282		if err == nil {
283			return nil
284		}
285		author, err := gi.ensurePerson(ctx, repo, item.ReopenedEvent.Actor)
286		if err != nil {
287			return err
288		}
289		op, err := b.OpenRaw(
290			author,
291			item.ReopenedEvent.CreatedAt.Unix(),
292			map[string]string{metaKeyGithubId: id},
293		)
294
295		if err != nil {
296			return err
297		}
298
299		gi.out <- core.NewImportStatusChange(op.Id())
300		return nil
301
302	case "RenamedTitleEvent":
303		id := parseId(item.RenamedTitleEvent.Id)
304		_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, id)
305		if err != cache.ErrNoMatchingOp {
306			return err
307		}
308		if err == nil {
309			return nil
310		}
311		author, err := gi.ensurePerson(ctx, repo, item.RenamedTitleEvent.Actor)
312		if err != nil {
313			return err
314		}
315
316		// At Github there exist issues with seemingly empty titles. An example is
317		// https://github.com/NixOS/nixpkgs/issues/72730 .
318		// The title provided by the GraphQL API actually consists of a space followed
319		// by a zero width space (U+200B). This title would cause the NewBugRaw()
320		// function to return an error: empty title.
321		title := string(item.RenamedTitleEvent.CurrentTitle)
322		if title == " \u200b" { // U+200B == zero width space
323			title = EMPTY_TITLE_PLACEHOLDER
324		}
325
326		op, err := b.SetTitleRaw(
327			author,
328			item.RenamedTitleEvent.CreatedAt.Unix(),
329			title,
330			map[string]string{metaKeyGithubId: id},
331		)
332		if err != nil {
333			return err
334		}
335
336		gi.out <- core.NewImportTitleEdition(op.Id())
337		return nil
338	}
339
340	return nil
341}
342
343func (gi *githubImporter) ensureComment(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, comment *issueComment) error {
344	author, err := gi.ensurePerson(ctx, repo, comment.Author)
345	if err != nil {
346		return err
347	}
348
349	targetOpID, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(comment.Id))
350	if err != nil && err != cache.ErrNoMatchingOp {
351		// real error
352		return err
353	}
354	firstEdit, hasEdit := <-gi.mediator.CommentEdits(comment)
355	if err == cache.ErrNoMatchingOp {
356		var textInput string
357		if hasEdit {
358			// use the first comment edit: it represents the comment creation itself
359			textInput = string(*firstEdit.Diff)
360		} else {
361			// if there are not comment edits, then the comment struct holds the comment creation
362			textInput = string(comment.Body)
363		}
364		cleanText, err := text.Cleanup(textInput)
365		if err != nil {
366			return err
367		}
368
369		// add comment operation
370		op, err := b.AddCommentRaw(
371			author,
372			comment.CreatedAt.Unix(),
373			cleanText,
374			nil,
375			map[string]string{
376				metaKeyGithubId:  parseId(comment.Id),
377				metaKeyGithubUrl: comment.Url.String(),
378			},
379		)
380		if err != nil {
381			return err
382		}
383
384		gi.out <- core.NewImportComment(op.Id())
385		targetOpID = op.Id()
386	}
387	if targetOpID == "" {
388		return fmt.Errorf("finding or creating issue comment")
389	}
390	// process remaining comment edits, if they exist
391	for edit := range gi.mediator.CommentEdits(comment) {
392		// ensure editor identity
393		_, err := gi.ensurePerson(ctx, repo, edit.Editor)
394		if err != nil {
395			return err
396		}
397
398		err = gi.ensureCommentEdit(ctx, repo, b, targetOpID, edit)
399		if err != nil {
400			return err
401		}
402	}
403	return nil
404}
405
406func (gi *githubImporter) ensureCommentEdit(ctx context.Context, repo *cache.RepoCache, b *cache.BugCache, target entity.Id, edit userContentEdit) error {
407	_, err := b.ResolveOperationWithMetadata(metaKeyGithubId, parseId(edit.Id))
408	if err == nil {
409		return nil
410	}
411	if err != cache.ErrNoMatchingOp {
412		// real error
413		return err
414	}
415
416	editor, err := gi.ensurePerson(ctx, repo, edit.Editor)
417	if err != nil {
418		return err
419	}
420
421	switch {
422	case edit.DeletedAt != nil:
423		// comment deletion, not supported yet
424		return nil
425
426	case edit.DeletedAt == nil:
427
428		cleanText, err := text.Cleanup(string(*edit.Diff))
429		if err != nil {
430			return err
431		}
432
433		// comment edition
434		op, err := b.EditCommentRaw(
435			editor,
436			edit.CreatedAt.Unix(),
437			target,
438			cleanText,
439			map[string]string{
440				metaKeyGithubId: parseId(edit.Id),
441			},
442		)
443
444		if err != nil {
445			return err
446		}
447
448		gi.out <- core.NewImportCommentEdition(op.Id())
449		return nil
450	}
451	return nil
452}
453
454// ensurePerson create a bug.Person from the Github data
455func (gi *githubImporter) ensurePerson(ctx context.Context, repo *cache.RepoCache, actor *actor) (*cache.IdentityCache, error) {
456	// When a user has been deleted, Github return a null actor, while displaying a profile named "ghost"
457	// in it's UI. So we need a special case to get it.
458	if actor == nil {
459		return gi.getGhost(ctx, repo)
460	}
461
462	// Look first in the cache
463	i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, string(actor.Login))
464	if err == nil {
465		return i, nil
466	}
467	if entity.IsErrMultipleMatch(err) {
468		return nil, err
469	}
470
471	// importing a new identity
472	var name string
473	var email string
474
475	switch actor.Typename {
476	case "User":
477		if actor.User.Name != nil {
478			name = string(*(actor.User.Name))
479		}
480		email = string(actor.User.Email)
481	case "Organization":
482		if actor.Organization.Name != nil {
483			name = string(*(actor.Organization.Name))
484		}
485		if actor.Organization.Email != nil {
486			email = string(*(actor.Organization.Email))
487		}
488	case "Bot":
489	}
490
491	// Name is not necessarily set, fallback to login as a name is required in the identity
492	if name == "" {
493		name = string(actor.Login)
494	}
495
496	i, err = repo.NewIdentityRaw(
497		name,
498		email,
499		string(actor.Login),
500		string(actor.AvatarUrl),
501		map[string]string{
502			metaKeyGithubLogin: string(actor.Login),
503		},
504	)
505
506	if err != nil {
507		return nil, err
508	}
509
510	gi.out <- core.NewImportIdentity(i.Id())
511	return i, nil
512}
513
514func (gi *githubImporter) getGhost(ctx context.Context, repo *cache.RepoCache) (*cache.IdentityCache, error) {
515	loginName := "ghost"
516	// Look first in the cache
517	i, err := repo.ResolveIdentityImmutableMetadata(metaKeyGithubLogin, loginName)
518	if err == nil {
519		return i, nil
520	}
521	if entity.IsErrMultipleMatch(err) {
522		return nil, err
523	}
524	user, err := gi.mediator.User(ctx, loginName)
525	userName := ""
526	if user.Name != nil {
527		userName = string(*user.Name)
528	}
529	return repo.NewIdentityRaw(
530		userName,
531		"",
532		string(user.Login),
533		string(user.AvatarUrl),
534		map[string]string{
535			metaKeyGithubLogin: string(user.Login),
536		},
537	)
538}
539
540// parseId converts the unusable githubv4.ID (an interface{}) into a string
541func parseId(id githubv4.ID) string {
542	return fmt.Sprintf("%v", id)
543}