1// Copyright 2018 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package packages
6
7// See doc.go for package documentation and implementation notes.
8
9import (
10 "context"
11 "encoding/json"
12 "fmt"
13 "go/ast"
14 "go/parser"
15 "go/scanner"
16 "go/token"
17 "go/types"
18 "io/ioutil"
19 "log"
20 "os"
21 "path/filepath"
22 "strings"
23 "sync"
24
25 "golang.org/x/tools/go/gcexportdata"
26)
27
28// A LoadMode specifies the amount of detail to return when loading.
29// Higher-numbered modes cause Load to return more information,
30// but may be slower. Load may return more information than requested.
31type LoadMode int
32
33const (
34 // The following constants are used to specify which fields of the Package
35 // should be filled when loading is done. As a special case to provide
36 // backwards compatibility, a LoadMode of 0 is equivalent to LoadFiles.
37 // For all other LoadModes, the bits below specify which fields will be filled
38 // in the result packages.
39 // WARNING: This part of the go/packages API is EXPERIMENTAL. It might
40 // be changed or removed up until April 15 2019. After that date it will
41 // be frozen.
42 // TODO(matloob): Remove this comment on April 15.
43
44 // ID and Errors (if present) will always be filled.
45
46 // NeedName adds Name and PkgPath.
47 NeedName LoadMode = 1 << iota
48
49 // NeedFiles adds GoFiles and OtherFiles.
50 NeedFiles
51
52 // NeedCompiledGoFiles adds CompiledGoFiles.
53 NeedCompiledGoFiles
54
55 // NeedImports adds Imports. If NeedDeps is not set, the Imports field will contain
56 // "placeholder" Packages with only the ID set.
57 NeedImports
58
59 // NeedDeps adds the fields requested by the LoadMode in the packages in Imports. If NeedImports
60 // is not set NeedDeps has no effect.
61 NeedDeps
62
63 // NeedExportsFile adds ExportsFile.
64 NeedExportsFile
65
66 // NeedTypes adds Types, Fset, and IllTyped.
67 NeedTypes
68
69 // NeedSyntax adds Syntax.
70 NeedSyntax
71
72 // NeedTypesInfo adds TypesInfo.
73 NeedTypesInfo
74
75 // NeedTypesSizes adds TypesSizes.
76 NeedTypesSizes
77)
78
79const (
80 // LoadFiles finds the packages and computes their source file lists.
81 // Package fields: ID, Name, Errors, GoFiles, CompiledGoFiles, and OtherFiles.
82 LoadFiles = NeedName | NeedFiles | NeedCompiledGoFiles
83
84 // LoadImports adds import information for each package
85 // and its dependencies.
86 // Package fields added: Imports.
87 LoadImports = LoadFiles | NeedImports | NeedDeps
88
89 // LoadTypes adds type information for package-level
90 // declarations in the packages matching the patterns.
91 // Package fields added: Types, TypesSizes, Fset, and IllTyped.
92 // This mode uses type information provided by the build system when
93 // possible, and may fill in the ExportFile field.
94 LoadTypes = LoadImports | NeedTypes | NeedTypesSizes
95
96 // LoadSyntax adds typed syntax trees for the packages matching the patterns.
97 // Package fields added: Syntax, and TypesInfo, for direct pattern matches only.
98 LoadSyntax = LoadTypes | NeedSyntax | NeedTypesInfo
99
100 // LoadAllSyntax adds typed syntax trees for the packages matching the patterns
101 // and all dependencies.
102 // Package fields added: Types, Fset, IllTyped, Syntax, and TypesInfo,
103 // for all packages in the import graph.
104 LoadAllSyntax = LoadSyntax
105)
106
107// A Config specifies details about how packages should be loaded.
108// The zero value is a valid configuration.
109// Calls to Load do not modify this struct.
110type Config struct {
111 // Mode controls the level of information returned for each package.
112 Mode LoadMode
113
114 // Context specifies the context for the load operation.
115 // If the context is cancelled, the loader may stop early
116 // and return an ErrCancelled error.
117 // If Context is nil, the load cannot be cancelled.
118 Context context.Context
119
120 // Dir is the directory in which to run the build system's query tool
121 // that provides information about the packages.
122 // If Dir is empty, the tool is run in the current directory.
123 Dir string
124
125 // Env is the environment to use when invoking the build system's query tool.
126 // If Env is nil, the current environment is used.
127 // As in os/exec's Cmd, only the last value in the slice for
128 // each environment key is used. To specify the setting of only
129 // a few variables, append to the current environment, as in:
130 //
131 // opt.Env = append(os.Environ(), "GOOS=plan9", "GOARCH=386")
132 //
133 Env []string
134
135 // BuildFlags is a list of command-line flags to be passed through to
136 // the build system's query tool.
137 BuildFlags []string
138
139 // Fset provides source position information for syntax trees and types.
140 // If Fset is nil, Load will use a new fileset, but preserve Fset's value.
141 Fset *token.FileSet
142
143 // ParseFile is called to read and parse each file
144 // when preparing a package's type-checked syntax tree.
145 // It must be safe to call ParseFile simultaneously from multiple goroutines.
146 // If ParseFile is nil, the loader will uses parser.ParseFile.
147 //
148 // ParseFile should parse the source from src and use filename only for
149 // recording position information.
150 //
151 // An application may supply a custom implementation of ParseFile
152 // to change the effective file contents or the behavior of the parser,
153 // or to modify the syntax tree. For example, selectively eliminating
154 // unwanted function bodies can significantly accelerate type checking.
155 ParseFile func(fset *token.FileSet, filename string, src []byte) (*ast.File, error)
156
157 // If Tests is set, the loader includes not just the packages
158 // matching a particular pattern but also any related test packages,
159 // including test-only variants of the package and the test executable.
160 //
161 // For example, when using the go command, loading "fmt" with Tests=true
162 // returns four packages, with IDs "fmt" (the standard package),
163 // "fmt [fmt.test]" (the package as compiled for the test),
164 // "fmt_test" (the test functions from source files in package fmt_test),
165 // and "fmt.test" (the test binary).
166 //
167 // In build systems with explicit names for tests,
168 // setting Tests may have no effect.
169 Tests bool
170
171 // Overlay provides a mapping of absolute file paths to file contents.
172 // If the file with the given path already exists, the parser will use the
173 // alternative file contents provided by the map.
174 //
175 // Overlays provide incomplete support for when a given file doesn't
176 // already exist on disk. See the package doc above for more details.
177 Overlay map[string][]byte
178}
179
180// driver is the type for functions that query the build system for the
181// packages named by the patterns.
182type driver func(cfg *Config, patterns ...string) (*driverResponse, error)
183
184// driverResponse contains the results for a driver query.
185type driverResponse struct {
186 // Sizes, if not nil, is the types.Sizes to use when type checking.
187 Sizes *types.StdSizes
188
189 // Roots is the set of package IDs that make up the root packages.
190 // We have to encode this separately because when we encode a single package
191 // we cannot know if it is one of the roots as that requires knowledge of the
192 // graph it is part of.
193 Roots []string `json:",omitempty"`
194
195 // Packages is the full set of packages in the graph.
196 // The packages are not connected into a graph.
197 // The Imports if populated will be stubs that only have their ID set.
198 // Imports will be connected and then type and syntax information added in a
199 // later pass (see refine).
200 Packages []*Package
201}
202
203// Load loads and returns the Go packages named by the given patterns.
204//
205// Config specifies loading options;
206// nil behaves the same as an empty Config.
207//
208// Load returns an error if any of the patterns was invalid
209// as defined by the underlying build system.
210// It may return an empty list of packages without an error,
211// for instance for an empty expansion of a valid wildcard.
212// Errors associated with a particular package are recorded in the
213// corresponding Package's Errors list, and do not cause Load to
214// return an error. Clients may need to handle such errors before
215// proceeding with further analysis. The PrintErrors function is
216// provided for convenient display of all errors.
217func Load(cfg *Config, patterns ...string) ([]*Package, error) {
218 l := newLoader(cfg)
219 response, err := defaultDriver(&l.Config, patterns...)
220 if err != nil {
221 return nil, err
222 }
223 l.sizes = response.Sizes
224 return l.refine(response.Roots, response.Packages...)
225}
226
227// defaultDriver is a driver that looks for an external driver binary, and if
228// it does not find it falls back to the built in go list driver.
229func defaultDriver(cfg *Config, patterns ...string) (*driverResponse, error) {
230 driver := findExternalDriver(cfg)
231 if driver == nil {
232 driver = goListDriver
233 }
234 return driver(cfg, patterns...)
235}
236
237// A Package describes a loaded Go package.
238type Package struct {
239 // ID is a unique identifier for a package,
240 // in a syntax provided by the underlying build system.
241 //
242 // Because the syntax varies based on the build system,
243 // clients should treat IDs as opaque and not attempt to
244 // interpret them.
245 ID string
246
247 // Name is the package name as it appears in the package source code.
248 Name string
249
250 // PkgPath is the package path as used by the go/types package.
251 PkgPath string
252
253 // Errors contains any errors encountered querying the metadata
254 // of the package, or while parsing or type-checking its files.
255 Errors []Error
256
257 // GoFiles lists the absolute file paths of the package's Go source files.
258 GoFiles []string
259
260 // CompiledGoFiles lists the absolute file paths of the package's source
261 // files that were presented to the compiler.
262 // This may differ from GoFiles if files are processed before compilation.
263 CompiledGoFiles []string
264
265 // OtherFiles lists the absolute file paths of the package's non-Go source files,
266 // including assembly, C, C++, Fortran, Objective-C, SWIG, and so on.
267 OtherFiles []string
268
269 // ExportFile is the absolute path to a file containing type
270 // information for the package as provided by the build system.
271 ExportFile string
272
273 // Imports maps import paths appearing in the package's Go source files
274 // to corresponding loaded Packages.
275 Imports map[string]*Package
276
277 // Types provides type information for the package.
278 // Modes LoadTypes and above set this field for packages matching the
279 // patterns; type information for dependencies may be missing or incomplete.
280 // Mode LoadAllSyntax sets this field for all packages, including dependencies.
281 Types *types.Package
282
283 // Fset provides position information for Types, TypesInfo, and Syntax.
284 // It is set only when Types is set.
285 Fset *token.FileSet
286
287 // IllTyped indicates whether the package or any dependency contains errors.
288 // It is set only when Types is set.
289 IllTyped bool
290
291 // Syntax is the package's syntax trees, for the files listed in CompiledGoFiles.
292 //
293 // Mode LoadSyntax sets this field for packages matching the patterns.
294 // Mode LoadAllSyntax sets this field for all packages, including dependencies.
295 Syntax []*ast.File
296
297 // TypesInfo provides type information about the package's syntax trees.
298 // It is set only when Syntax is set.
299 TypesInfo *types.Info
300
301 // TypesSizes provides the effective size function for types in TypesInfo.
302 TypesSizes types.Sizes
303}
304
305// An Error describes a problem with a package's metadata, syntax, or types.
306type Error struct {
307 Pos string // "file:line:col" or "file:line" or "" or "-"
308 Msg string
309 Kind ErrorKind
310}
311
312// ErrorKind describes the source of the error, allowing the user to
313// differentiate between errors generated by the driver, the parser, or the
314// type-checker.
315type ErrorKind int
316
317const (
318 UnknownError ErrorKind = iota
319 ListError
320 ParseError
321 TypeError
322)
323
324func (err Error) Error() string {
325 pos := err.Pos
326 if pos == "" {
327 pos = "-" // like token.Position{}.String()
328 }
329 return pos + ": " + err.Msg
330}
331
332// flatPackage is the JSON form of Package
333// It drops all the type and syntax fields, and transforms the Imports
334//
335// TODO(adonovan): identify this struct with Package, effectively
336// publishing the JSON protocol.
337type flatPackage struct {
338 ID string
339 Name string `json:",omitempty"`
340 PkgPath string `json:",omitempty"`
341 Errors []Error `json:",omitempty"`
342 GoFiles []string `json:",omitempty"`
343 CompiledGoFiles []string `json:",omitempty"`
344 OtherFiles []string `json:",omitempty"`
345 ExportFile string `json:",omitempty"`
346 Imports map[string]string `json:",omitempty"`
347}
348
349// MarshalJSON returns the Package in its JSON form.
350// For the most part, the structure fields are written out unmodified, and
351// the type and syntax fields are skipped.
352// The imports are written out as just a map of path to package id.
353// The errors are written using a custom type that tries to preserve the
354// structure of error types we know about.
355//
356// This method exists to enable support for additional build systems. It is
357// not intended for use by clients of the API and we may change the format.
358func (p *Package) MarshalJSON() ([]byte, error) {
359 flat := &flatPackage{
360 ID: p.ID,
361 Name: p.Name,
362 PkgPath: p.PkgPath,
363 Errors: p.Errors,
364 GoFiles: p.GoFiles,
365 CompiledGoFiles: p.CompiledGoFiles,
366 OtherFiles: p.OtherFiles,
367 ExportFile: p.ExportFile,
368 }
369 if len(p.Imports) > 0 {
370 flat.Imports = make(map[string]string, len(p.Imports))
371 for path, ipkg := range p.Imports {
372 flat.Imports[path] = ipkg.ID
373 }
374 }
375 return json.Marshal(flat)
376}
377
378// UnmarshalJSON reads in a Package from its JSON format.
379// See MarshalJSON for details about the format accepted.
380func (p *Package) UnmarshalJSON(b []byte) error {
381 flat := &flatPackage{}
382 if err := json.Unmarshal(b, &flat); err != nil {
383 return err
384 }
385 *p = Package{
386 ID: flat.ID,
387 Name: flat.Name,
388 PkgPath: flat.PkgPath,
389 Errors: flat.Errors,
390 GoFiles: flat.GoFiles,
391 CompiledGoFiles: flat.CompiledGoFiles,
392 OtherFiles: flat.OtherFiles,
393 ExportFile: flat.ExportFile,
394 }
395 if len(flat.Imports) > 0 {
396 p.Imports = make(map[string]*Package, len(flat.Imports))
397 for path, id := range flat.Imports {
398 p.Imports[path] = &Package{ID: id}
399 }
400 }
401 return nil
402}
403
404func (p *Package) String() string { return p.ID }
405
406// loaderPackage augments Package with state used during the loading phase
407type loaderPackage struct {
408 *Package
409 importErrors map[string]error // maps each bad import to its error
410 loadOnce sync.Once
411 color uint8 // for cycle detection
412 needsrc bool // load from source (Mode >= LoadTypes)
413 needtypes bool // type information is either requested or depended on
414 initial bool // package was matched by a pattern
415}
416
417// loader holds the working state of a single call to load.
418type loader struct {
419 pkgs map[string]*loaderPackage
420 Config
421 sizes types.Sizes
422 exportMu sync.Mutex // enforces mutual exclusion of exportdata operations
423
424 // TODO(matloob): Add an implied mode here and use that instead of mode.
425 // Implied mode would contain all the fields we need the data for so we can
426 // get the actually requested fields. We'll zero them out before returning
427 // packages to the user. This will make it easier for us to get the conditions
428 // where we need certain modes right.
429}
430
431func newLoader(cfg *Config) *loader {
432 ld := &loader{}
433 if cfg != nil {
434 ld.Config = *cfg
435 }
436 if ld.Config.Mode == 0 {
437 ld.Config.Mode = LoadFiles // Preserve zero behavior of Mode for backwards compatibility.
438 }
439 if ld.Config.Env == nil {
440 ld.Config.Env = os.Environ()
441 }
442 if ld.Context == nil {
443 ld.Context = context.Background()
444 }
445 if ld.Dir == "" {
446 if dir, err := os.Getwd(); err == nil {
447 ld.Dir = dir
448 }
449 }
450
451 if ld.Mode&NeedTypes != 0 {
452 if ld.Fset == nil {
453 ld.Fset = token.NewFileSet()
454 }
455
456 // ParseFile is required even in LoadTypes mode
457 // because we load source if export data is missing.
458 if ld.ParseFile == nil {
459 ld.ParseFile = func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) {
460 var isrc interface{}
461 if src != nil {
462 isrc = src
463 }
464 const mode = parser.AllErrors | parser.ParseComments
465 return parser.ParseFile(fset, filename, isrc, mode)
466 }
467 }
468 }
469 return ld
470}
471
472// refine connects the supplied packages into a graph and then adds type and
473// and syntax information as requested by the LoadMode.
474func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
475 rootMap := make(map[string]int, len(roots))
476 for i, root := range roots {
477 rootMap[root] = i
478 }
479 ld.pkgs = make(map[string]*loaderPackage)
480 // first pass, fixup and build the map and roots
481 var initial = make([]*loaderPackage, len(roots))
482 for _, pkg := range list {
483 rootIndex := -1
484 if i, found := rootMap[pkg.ID]; found {
485 rootIndex = i
486 }
487 lpkg := &loaderPackage{
488 Package: pkg,
489 needtypes: (ld.Mode&(NeedTypes|NeedTypesInfo) != 0 && rootIndex < 0) || rootIndex >= 0,
490 needsrc: (ld.Mode&(NeedSyntax|NeedTypesInfo) != 0 && rootIndex < 0) || rootIndex >= 0 ||
491 len(ld.Overlay) > 0 || // Overlays can invalidate export data. TODO(matloob): make this check fine-grained based on dependencies on overlaid files
492 pkg.ExportFile == "" && pkg.PkgPath != "unsafe",
493 }
494 ld.pkgs[lpkg.ID] = lpkg
495 if rootIndex >= 0 {
496 initial[rootIndex] = lpkg
497 lpkg.initial = true
498 }
499 }
500 for i, root := range roots {
501 if initial[i] == nil {
502 return nil, fmt.Errorf("root package %v is missing", root)
503 }
504 }
505
506 // Materialize the import graph.
507
508 const (
509 white = 0 // new
510 grey = 1 // in progress
511 black = 2 // complete
512 )
513
514 // visit traverses the import graph, depth-first,
515 // and materializes the graph as Packages.Imports.
516 //
517 // Valid imports are saved in the Packages.Import map.
518 // Invalid imports (cycles and missing nodes) are saved in the importErrors map.
519 // Thus, even in the presence of both kinds of errors, the Import graph remains a DAG.
520 //
521 // visit returns whether the package needs src or has a transitive
522 // dependency on a package that does. These are the only packages
523 // for which we load source code.
524 var stack []*loaderPackage
525 var visit func(lpkg *loaderPackage) bool
526 var srcPkgs []*loaderPackage
527 visit = func(lpkg *loaderPackage) bool {
528 switch lpkg.color {
529 case black:
530 return lpkg.needsrc
531 case grey:
532 panic("internal error: grey node")
533 }
534 lpkg.color = grey
535 stack = append(stack, lpkg) // push
536 stubs := lpkg.Imports // the structure form has only stubs with the ID in the Imports
537 lpkg.Imports = make(map[string]*Package, len(stubs))
538 for importPath, ipkg := range stubs {
539 var importErr error
540 imp := ld.pkgs[ipkg.ID]
541 if imp == nil {
542 // (includes package "C" when DisableCgo)
543 importErr = fmt.Errorf("missing package: %q", ipkg.ID)
544 } else if imp.color == grey {
545 importErr = fmt.Errorf("import cycle: %s", stack)
546 }
547 if importErr != nil {
548 if lpkg.importErrors == nil {
549 lpkg.importErrors = make(map[string]error)
550 }
551 lpkg.importErrors[importPath] = importErr
552 continue
553 }
554
555 if visit(imp) {
556 lpkg.needsrc = true
557 }
558 lpkg.Imports[importPath] = imp.Package
559 }
560 if lpkg.needsrc {
561 srcPkgs = append(srcPkgs, lpkg)
562 }
563 if ld.Mode&NeedTypesSizes != 0 {
564 lpkg.TypesSizes = ld.sizes
565 }
566 stack = stack[:len(stack)-1] // pop
567 lpkg.color = black
568
569 return lpkg.needsrc
570 }
571
572 if ld.Mode&(NeedImports|NeedDeps) == 0 {
573 // We do this to drop the stub import packages that we are not even going to try to resolve.
574 for _, lpkg := range initial {
575 lpkg.Imports = nil
576 }
577 } else {
578 // For each initial package, create its import DAG.
579 for _, lpkg := range initial {
580 visit(lpkg)
581 }
582 }
583 if ld.Mode&NeedDeps != 0 { // TODO(matloob): This is only the case if NeedTypes is also set, right?
584 for _, lpkg := range srcPkgs {
585 // Complete type information is required for the
586 // immediate dependencies of each source package.
587 for _, ipkg := range lpkg.Imports {
588 imp := ld.pkgs[ipkg.ID]
589 imp.needtypes = true
590 }
591 }
592 }
593 // Load type data if needed, starting at
594 // the initial packages (roots of the import DAG).
595 if ld.Mode&NeedTypes != 0 {
596 var wg sync.WaitGroup
597 for _, lpkg := range initial {
598 wg.Add(1)
599 go func(lpkg *loaderPackage) {
600 ld.loadRecursive(lpkg)
601 wg.Done()
602 }(lpkg)
603 }
604 wg.Wait()
605 }
606
607 result := make([]*Package, len(initial))
608 importPlaceholders := make(map[string]*Package)
609 for i, lpkg := range initial {
610 result[i] = lpkg.Package
611 }
612 for i := range ld.pkgs {
613 // Clear all unrequested fields, for extra de-Hyrum-ization.
614 if ld.Mode&NeedName == 0 {
615 ld.pkgs[i].Name = ""
616 ld.pkgs[i].PkgPath = ""
617 }
618 if ld.Mode&NeedFiles == 0 {
619 ld.pkgs[i].GoFiles = nil
620 ld.pkgs[i].OtherFiles = nil
621 }
622 if ld.Mode&NeedCompiledGoFiles == 0 {
623 ld.pkgs[i].CompiledGoFiles = nil
624 }
625 if ld.Mode&NeedImports == 0 {
626 ld.pkgs[i].Imports = nil
627 }
628 if ld.Mode&NeedExportsFile == 0 {
629 ld.pkgs[i].ExportFile = ""
630 }
631 if ld.Mode&NeedTypes == 0 {
632 ld.pkgs[i].Types = nil
633 ld.pkgs[i].Fset = nil
634 ld.pkgs[i].IllTyped = false
635 }
636 if ld.Mode&NeedSyntax == 0 {
637 ld.pkgs[i].Syntax = nil
638 }
639 if ld.Mode&NeedTypesInfo == 0 {
640 ld.pkgs[i].TypesInfo = nil
641 }
642 if ld.Mode&NeedTypesSizes == 0 {
643 ld.pkgs[i].TypesSizes = nil
644 }
645 if ld.Mode&NeedDeps == 0 {
646 for j, pkg := range ld.pkgs[i].Imports {
647 ph, ok := importPlaceholders[pkg.ID]
648 if !ok {
649 ph = &Package{ID: pkg.ID}
650 importPlaceholders[pkg.ID] = ph
651 }
652 ld.pkgs[i].Imports[j] = ph
653 }
654 }
655 }
656 return result, nil
657}
658
659// loadRecursive loads the specified package and its dependencies,
660// recursively, in parallel, in topological order.
661// It is atomic and idempotent.
662// Precondition: ld.Mode&NeedTypes.
663func (ld *loader) loadRecursive(lpkg *loaderPackage) {
664 lpkg.loadOnce.Do(func() {
665 // Load the direct dependencies, in parallel.
666 var wg sync.WaitGroup
667 for _, ipkg := range lpkg.Imports {
668 imp := ld.pkgs[ipkg.ID]
669 wg.Add(1)
670 go func(imp *loaderPackage) {
671 ld.loadRecursive(imp)
672 wg.Done()
673 }(imp)
674 }
675 wg.Wait()
676
677 ld.loadPackage(lpkg)
678 })
679}
680
681// loadPackage loads the specified package.
682// It must be called only once per Package,
683// after immediate dependencies are loaded.
684// Precondition: ld.Mode >= LoadTypes.
685func (ld *loader) loadPackage(lpkg *loaderPackage) {
686 if lpkg.PkgPath == "unsafe" {
687 // Fill in the blanks to avoid surprises.
688 lpkg.Types = types.Unsafe
689 lpkg.Fset = ld.Fset
690 lpkg.Syntax = []*ast.File{}
691 lpkg.TypesInfo = new(types.Info)
692 lpkg.TypesSizes = ld.sizes
693 return
694 }
695
696 // Call NewPackage directly with explicit name.
697 // This avoids skew between golist and go/types when the files'
698 // package declarations are inconsistent.
699 lpkg.Types = types.NewPackage(lpkg.PkgPath, lpkg.Name)
700 lpkg.Fset = ld.Fset
701
702 // Subtle: we populate all Types fields with an empty Package
703 // before loading export data so that export data processing
704 // never has to create a types.Package for an indirect dependency,
705 // which would then require that such created packages be explicitly
706 // inserted back into the Import graph as a final step after export data loading.
707 // The Diamond test exercises this case.
708 if !lpkg.needtypes {
709 return
710 }
711 if !lpkg.needsrc {
712 ld.loadFromExportData(lpkg)
713 return // not a source package, don't get syntax trees
714 }
715
716 appendError := func(err error) {
717 // Convert various error types into the one true Error.
718 var errs []Error
719 switch err := err.(type) {
720 case Error:
721 // from driver
722 errs = append(errs, err)
723
724 case *os.PathError:
725 // from parser
726 errs = append(errs, Error{
727 Pos: err.Path + ":1",
728 Msg: err.Err.Error(),
729 Kind: ParseError,
730 })
731
732 case scanner.ErrorList:
733 // from parser
734 for _, err := range err {
735 errs = append(errs, Error{
736 Pos: err.Pos.String(),
737 Msg: err.Msg,
738 Kind: ParseError,
739 })
740 }
741
742 case types.Error:
743 // from type checker
744 errs = append(errs, Error{
745 Pos: err.Fset.Position(err.Pos).String(),
746 Msg: err.Msg,
747 Kind: TypeError,
748 })
749
750 default:
751 // unexpected impoverished error from parser?
752 errs = append(errs, Error{
753 Pos: "-",
754 Msg: err.Error(),
755 Kind: UnknownError,
756 })
757
758 // If you see this error message, please file a bug.
759 log.Printf("internal error: error %q (%T) without position", err, err)
760 }
761
762 lpkg.Errors = append(lpkg.Errors, errs...)
763 }
764
765 files, errs := ld.parseFiles(lpkg.CompiledGoFiles)
766 for _, err := range errs {
767 appendError(err)
768 }
769
770 lpkg.Syntax = files
771
772 lpkg.TypesInfo = &types.Info{
773 Types: make(map[ast.Expr]types.TypeAndValue),
774 Defs: make(map[*ast.Ident]types.Object),
775 Uses: make(map[*ast.Ident]types.Object),
776 Implicits: make(map[ast.Node]types.Object),
777 Scopes: make(map[ast.Node]*types.Scope),
778 Selections: make(map[*ast.SelectorExpr]*types.Selection),
779 }
780 lpkg.TypesSizes = ld.sizes
781
782 importer := importerFunc(func(path string) (*types.Package, error) {
783 if path == "unsafe" {
784 return types.Unsafe, nil
785 }
786
787 // The imports map is keyed by import path.
788 ipkg := lpkg.Imports[path]
789 if ipkg == nil {
790 if err := lpkg.importErrors[path]; err != nil {
791 return nil, err
792 }
793 // There was skew between the metadata and the
794 // import declarations, likely due to an edit
795 // race, or because the ParseFile feature was
796 // used to supply alternative file contents.
797 return nil, fmt.Errorf("no metadata for %s", path)
798 }
799
800 if ipkg.Types != nil && ipkg.Types.Complete() {
801 return ipkg.Types, nil
802 }
803 log.Fatalf("internal error: nil Pkg importing %q from %q", path, lpkg)
804 panic("unreachable")
805 })
806
807 // type-check
808 tc := &types.Config{
809 Importer: importer,
810
811 // Type-check bodies of functions only in non-initial packages.
812 // Example: for import graph A->B->C and initial packages {A,C},
813 // we can ignore function bodies in B.
814 IgnoreFuncBodies: (ld.Mode&(NeedDeps|NeedTypesInfo) == 0) && !lpkg.initial,
815
816 Error: appendError,
817 Sizes: ld.sizes,
818 }
819 types.NewChecker(tc, ld.Fset, lpkg.Types, lpkg.TypesInfo).Files(lpkg.Syntax)
820
821 lpkg.importErrors = nil // no longer needed
822
823 // If !Cgo, the type-checker uses FakeImportC mode, so
824 // it doesn't invoke the importer for import "C",
825 // nor report an error for the import,
826 // or for any undefined C.f reference.
827 // We must detect this explicitly and correctly
828 // mark the package as IllTyped (by reporting an error).
829 // TODO(adonovan): if these errors are annoying,
830 // we could just set IllTyped quietly.
831 if tc.FakeImportC {
832 outer:
833 for _, f := range lpkg.Syntax {
834 for _, imp := range f.Imports {
835 if imp.Path.Value == `"C"` {
836 err := types.Error{Fset: ld.Fset, Pos: imp.Pos(), Msg: `import "C" ignored`}
837 appendError(err)
838 break outer
839 }
840 }
841 }
842 }
843
844 // Record accumulated errors.
845 illTyped := len(lpkg.Errors) > 0
846 if !illTyped {
847 for _, imp := range lpkg.Imports {
848 if imp.IllTyped {
849 illTyped = true
850 break
851 }
852 }
853 }
854 lpkg.IllTyped = illTyped
855}
856
857// An importFunc is an implementation of the single-method
858// types.Importer interface based on a function value.
859type importerFunc func(path string) (*types.Package, error)
860
861func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) }
862
863// We use a counting semaphore to limit
864// the number of parallel I/O calls per process.
865var ioLimit = make(chan bool, 20)
866
867// parseFiles reads and parses the Go source files and returns the ASTs
868// of the ones that could be at least partially parsed, along with a
869// list of I/O and parse errors encountered.
870//
871// Because files are scanned in parallel, the token.Pos
872// positions of the resulting ast.Files are not ordered.
873//
874func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) {
875 var wg sync.WaitGroup
876 n := len(filenames)
877 parsed := make([]*ast.File, n)
878 errors := make([]error, n)
879 for i, file := range filenames {
880 if ld.Config.Context.Err() != nil {
881 parsed[i] = nil
882 errors[i] = ld.Config.Context.Err()
883 continue
884 }
885 wg.Add(1)
886 go func(i int, filename string) {
887 ioLimit <- true // wait
888 // ParseFile may return both an AST and an error.
889 var src []byte
890 for f, contents := range ld.Config.Overlay {
891 if sameFile(f, filename) {
892 src = contents
893 }
894 }
895 var err error
896 if src == nil {
897 src, err = ioutil.ReadFile(filename)
898 }
899 if err != nil {
900 parsed[i], errors[i] = nil, err
901 } else {
902 parsed[i], errors[i] = ld.ParseFile(ld.Fset, filename, src)
903 }
904 <-ioLimit // signal
905 wg.Done()
906 }(i, file)
907 }
908 wg.Wait()
909
910 // Eliminate nils, preserving order.
911 var o int
912 for _, f := range parsed {
913 if f != nil {
914 parsed[o] = f
915 o++
916 }
917 }
918 parsed = parsed[:o]
919
920 o = 0
921 for _, err := range errors {
922 if err != nil {
923 errors[o] = err
924 o++
925 }
926 }
927 errors = errors[:o]
928
929 return parsed, errors
930}
931
932// sameFile returns true if x and y have the same basename and denote
933// the same file.
934//
935func sameFile(x, y string) bool {
936 if x == y {
937 // It could be the case that y doesn't exist.
938 // For instance, it may be an overlay file that
939 // hasn't been written to disk. To handle that case
940 // let x == y through. (We added the exact absolute path
941 // string to the CompiledGoFiles list, so the unwritten
942 // overlay case implies x==y.)
943 return true
944 }
945 if strings.EqualFold(filepath.Base(x), filepath.Base(y)) { // (optimisation)
946 if xi, err := os.Stat(x); err == nil {
947 if yi, err := os.Stat(y); err == nil {
948 return os.SameFile(xi, yi)
949 }
950 }
951 }
952 return false
953}
954
955// loadFromExportData returns type information for the specified
956// package, loading it from an export data file on the first request.
957func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error) {
958 if lpkg.PkgPath == "" {
959 log.Fatalf("internal error: Package %s has no PkgPath", lpkg)
960 }
961
962 // Because gcexportdata.Read has the potential to create or
963 // modify the types.Package for each node in the transitive
964 // closure of dependencies of lpkg, all exportdata operations
965 // must be sequential. (Finer-grained locking would require
966 // changes to the gcexportdata API.)
967 //
968 // The exportMu lock guards the Package.Pkg field and the
969 // types.Package it points to, for each Package in the graph.
970 //
971 // Not all accesses to Package.Pkg need to be protected by exportMu:
972 // graph ordering ensures that direct dependencies of source
973 // packages are fully loaded before the importer reads their Pkg field.
974 ld.exportMu.Lock()
975 defer ld.exportMu.Unlock()
976
977 if tpkg := lpkg.Types; tpkg != nil && tpkg.Complete() {
978 return tpkg, nil // cache hit
979 }
980
981 lpkg.IllTyped = true // fail safe
982
983 if lpkg.ExportFile == "" {
984 // Errors while building export data will have been printed to stderr.
985 return nil, fmt.Errorf("no export data file")
986 }
987 f, err := os.Open(lpkg.ExportFile)
988 if err != nil {
989 return nil, err
990 }
991 defer f.Close()
992
993 // Read gc export data.
994 //
995 // We don't currently support gccgo export data because all
996 // underlying workspaces use the gc toolchain. (Even build
997 // systems that support gccgo don't use it for workspace
998 // queries.)
999 r, err := gcexportdata.NewReader(f)
1000 if err != nil {
1001 return nil, fmt.Errorf("reading %s: %v", lpkg.ExportFile, err)
1002 }
1003
1004 // Build the view.
1005 //
1006 // The gcexportdata machinery has no concept of package ID.
1007 // It identifies packages by their PkgPath, which although not
1008 // globally unique is unique within the scope of one invocation
1009 // of the linker, type-checker, or gcexportdata.
1010 //
1011 // So, we must build a PkgPath-keyed view of the global
1012 // (conceptually ID-keyed) cache of packages and pass it to
1013 // gcexportdata. The view must contain every existing
1014 // package that might possibly be mentioned by the
1015 // current package---its transitive closure.
1016 //
1017 // In loadPackage, we unconditionally create a types.Package for
1018 // each dependency so that export data loading does not
1019 // create new ones.
1020 //
1021 // TODO(adonovan): it would be simpler and more efficient
1022 // if the export data machinery invoked a callback to
1023 // get-or-create a package instead of a map.
1024 //
1025 view := make(map[string]*types.Package) // view seen by gcexportdata
1026 seen := make(map[*loaderPackage]bool) // all visited packages
1027 var visit func(pkgs map[string]*Package)
1028 visit = func(pkgs map[string]*Package) {
1029 for _, p := range pkgs {
1030 lpkg := ld.pkgs[p.ID]
1031 if !seen[lpkg] {
1032 seen[lpkg] = true
1033 view[lpkg.PkgPath] = lpkg.Types
1034 visit(lpkg.Imports)
1035 }
1036 }
1037 }
1038 visit(lpkg.Imports)
1039
1040 viewLen := len(view) + 1 // adding the self package
1041 // Parse the export data.
1042 // (May modify incomplete packages in view but not create new ones.)
1043 tpkg, err := gcexportdata.Read(r, ld.Fset, view, lpkg.PkgPath)
1044 if err != nil {
1045 return nil, fmt.Errorf("reading %s: %v", lpkg.ExportFile, err)
1046 }
1047 if viewLen != len(view) {
1048 log.Fatalf("Unexpected package creation during export data loading")
1049 }
1050
1051 lpkg.Types = tpkg
1052 lpkg.IllTyped = false
1053
1054 return tpkg, nil
1055}
1056
1057func usesExportData(cfg *Config) bool {
1058 return cfg.Mode&NeedExportsFile != 0 || cfg.Mode&NeedTypes != 0 && cfg.Mode&NeedTypesInfo == 0
1059}