1// Copyright 2018 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5//go:build 386 || amd64 || amd64p32
6
7package cpu
8
9import "runtime"
10
11const cacheLineSize = 64
12
13func initOptions() {
14 options = []option{
15 {Name: "adx", Feature: &X86.HasADX},
16 {Name: "aes", Feature: &X86.HasAES},
17 {Name: "avx", Feature: &X86.HasAVX},
18 {Name: "avx2", Feature: &X86.HasAVX2},
19 {Name: "avx512", Feature: &X86.HasAVX512},
20 {Name: "avx512f", Feature: &X86.HasAVX512F},
21 {Name: "avx512cd", Feature: &X86.HasAVX512CD},
22 {Name: "avx512er", Feature: &X86.HasAVX512ER},
23 {Name: "avx512pf", Feature: &X86.HasAVX512PF},
24 {Name: "avx512vl", Feature: &X86.HasAVX512VL},
25 {Name: "avx512bw", Feature: &X86.HasAVX512BW},
26 {Name: "avx512dq", Feature: &X86.HasAVX512DQ},
27 {Name: "avx512ifma", Feature: &X86.HasAVX512IFMA},
28 {Name: "avx512vbmi", Feature: &X86.HasAVX512VBMI},
29 {Name: "avx512vnniw", Feature: &X86.HasAVX5124VNNIW},
30 {Name: "avx5124fmaps", Feature: &X86.HasAVX5124FMAPS},
31 {Name: "avx512vpopcntdq", Feature: &X86.HasAVX512VPOPCNTDQ},
32 {Name: "avx512vpclmulqdq", Feature: &X86.HasAVX512VPCLMULQDQ},
33 {Name: "avx512vnni", Feature: &X86.HasAVX512VNNI},
34 {Name: "avx512gfni", Feature: &X86.HasAVX512GFNI},
35 {Name: "avx512vaes", Feature: &X86.HasAVX512VAES},
36 {Name: "avx512vbmi2", Feature: &X86.HasAVX512VBMI2},
37 {Name: "avx512bitalg", Feature: &X86.HasAVX512BITALG},
38 {Name: "avx512bf16", Feature: &X86.HasAVX512BF16},
39 {Name: "amxtile", Feature: &X86.HasAMXTile},
40 {Name: "amxint8", Feature: &X86.HasAMXInt8},
41 {Name: "amxbf16", Feature: &X86.HasAMXBF16},
42 {Name: "bmi1", Feature: &X86.HasBMI1},
43 {Name: "bmi2", Feature: &X86.HasBMI2},
44 {Name: "cx16", Feature: &X86.HasCX16},
45 {Name: "erms", Feature: &X86.HasERMS},
46 {Name: "fma", Feature: &X86.HasFMA},
47 {Name: "osxsave", Feature: &X86.HasOSXSAVE},
48 {Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ},
49 {Name: "popcnt", Feature: &X86.HasPOPCNT},
50 {Name: "rdrand", Feature: &X86.HasRDRAND},
51 {Name: "rdseed", Feature: &X86.HasRDSEED},
52 {Name: "sse3", Feature: &X86.HasSSE3},
53 {Name: "sse41", Feature: &X86.HasSSE41},
54 {Name: "sse42", Feature: &X86.HasSSE42},
55 {Name: "ssse3", Feature: &X86.HasSSSE3},
56 {Name: "avxifma", Feature: &X86.HasAVXIFMA},
57 {Name: "avxvnni", Feature: &X86.HasAVXVNNI},
58 {Name: "avxvnniint8", Feature: &X86.HasAVXVNNIInt8},
59
60 // These capabilities should always be enabled on amd64:
61 {Name: "sse2", Feature: &X86.HasSSE2, Required: runtime.GOARCH == "amd64"},
62 }
63}
64
65func archInit() {
66
67 Initialized = true
68
69 maxID, _, _, _ := cpuid(0, 0)
70
71 if maxID < 1 {
72 return
73 }
74
75 _, _, ecx1, edx1 := cpuid(1, 0)
76 X86.HasSSE2 = isSet(26, edx1)
77
78 X86.HasSSE3 = isSet(0, ecx1)
79 X86.HasPCLMULQDQ = isSet(1, ecx1)
80 X86.HasSSSE3 = isSet(9, ecx1)
81 X86.HasFMA = isSet(12, ecx1)
82 X86.HasCX16 = isSet(13, ecx1)
83 X86.HasSSE41 = isSet(19, ecx1)
84 X86.HasSSE42 = isSet(20, ecx1)
85 X86.HasPOPCNT = isSet(23, ecx1)
86 X86.HasAES = isSet(25, ecx1)
87 X86.HasOSXSAVE = isSet(27, ecx1)
88 X86.HasRDRAND = isSet(30, ecx1)
89
90 var osSupportsAVX, osSupportsAVX512 bool
91 // For XGETBV, OSXSAVE bit is required and sufficient.
92 if X86.HasOSXSAVE {
93 eax, _ := xgetbv()
94 // Check if XMM and YMM registers have OS support.
95 osSupportsAVX = isSet(1, eax) && isSet(2, eax)
96
97 if runtime.GOOS == "darwin" {
98 // Darwin requires special AVX512 checks, see cpu_darwin_x86.go
99 osSupportsAVX512 = osSupportsAVX && darwinSupportsAVX512()
100 } else {
101 // Check if OPMASK and ZMM registers have OS support.
102 osSupportsAVX512 = osSupportsAVX && isSet(5, eax) && isSet(6, eax) && isSet(7, eax)
103 }
104 }
105
106 X86.HasAVX = isSet(28, ecx1) && osSupportsAVX
107
108 if maxID < 7 {
109 return
110 }
111
112 eax7, ebx7, ecx7, edx7 := cpuid(7, 0)
113 X86.HasBMI1 = isSet(3, ebx7)
114 X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX
115 X86.HasBMI2 = isSet(8, ebx7)
116 X86.HasERMS = isSet(9, ebx7)
117 X86.HasRDSEED = isSet(18, ebx7)
118 X86.HasADX = isSet(19, ebx7)
119
120 X86.HasAVX512 = isSet(16, ebx7) && osSupportsAVX512 // Because avx-512 foundation is the core required extension
121 if X86.HasAVX512 {
122 X86.HasAVX512F = true
123 X86.HasAVX512CD = isSet(28, ebx7)
124 X86.HasAVX512ER = isSet(27, ebx7)
125 X86.HasAVX512PF = isSet(26, ebx7)
126 X86.HasAVX512VL = isSet(31, ebx7)
127 X86.HasAVX512BW = isSet(30, ebx7)
128 X86.HasAVX512DQ = isSet(17, ebx7)
129 X86.HasAVX512IFMA = isSet(21, ebx7)
130 X86.HasAVX512VBMI = isSet(1, ecx7)
131 X86.HasAVX5124VNNIW = isSet(2, edx7)
132 X86.HasAVX5124FMAPS = isSet(3, edx7)
133 X86.HasAVX512VPOPCNTDQ = isSet(14, ecx7)
134 X86.HasAVX512VPCLMULQDQ = isSet(10, ecx7)
135 X86.HasAVX512VNNI = isSet(11, ecx7)
136 X86.HasAVX512GFNI = isSet(8, ecx7)
137 X86.HasAVX512VAES = isSet(9, ecx7)
138 X86.HasAVX512VBMI2 = isSet(6, ecx7)
139 X86.HasAVX512BITALG = isSet(12, ecx7)
140 }
141
142 X86.HasAMXTile = isSet(24, edx7)
143 X86.HasAMXInt8 = isSet(25, edx7)
144 X86.HasAMXBF16 = isSet(22, edx7)
145
146 // These features depend on the second level of extended features.
147 if eax7 >= 1 {
148 eax71, _, _, edx71 := cpuid(7, 1)
149 if X86.HasAVX512 {
150 X86.HasAVX512BF16 = isSet(5, eax71)
151 }
152 if X86.HasAVX {
153 X86.HasAVXIFMA = isSet(23, eax71)
154 X86.HasAVXVNNI = isSet(4, eax71)
155 X86.HasAVXVNNIInt8 = isSet(4, edx71)
156 }
157 }
158}
159
160func isSet(bitpos uint, value uint32) bool {
161 return value&(1<<bitpos) != 0
162}