Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 1 | /* cpu_features.c -- Processor features detection. |
| 2 | * |
Avi Drissman | cbb6b98 | 2022-09-27 19:16:57 +0000 | [diff] [blame] | 3 | * Copyright 2018 The Chromium Authors |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 4 | * Use of this source code is governed by a BSD-style license that can be |
| 5 | * found in the Chromium source repository LICENSE file. |
| 6 | */ |
| 7 | |
| 8 | #include "cpu_features.h" |
| 9 | #include "zutil.h" |
| 10 | |
| 11 | #include <stdint.h> |
| 12 | #if defined(_MSC_VER) |
| 13 | #include <intrin.h> |
| 14 | #elif defined(ADLER32_SIMD_SSSE3) |
| 15 | #include <cpuid.h> |
| 16 | #endif |
| 17 | |
| 18 | /* TODO(cavalcantii): remove checks for x86_flags on deflate. |
| 19 | */ |
Mark Mentovai | 8603eee | 2020-06-28 07:59:33 +0000 | [diff] [blame] | 20 | #if defined(ARMV8_OS_MACOS) |
Adenilson Cavalcanti | 05e137d | 2022-09-09 18:14:22 +0000 | [diff] [blame] | 21 | /* Crypto extensions (crc32/pmull) are a baseline feature in ARMv8.1-A, and |
| 22 | * OSX running on arm64 is new enough that these can be assumed without |
| 23 | * runtime detection. |
| 24 | */ |
Mark Mentovai | 8603eee | 2020-06-28 07:59:33 +0000 | [diff] [blame] | 25 | int ZLIB_INTERNAL arm_cpu_enable_crc32 = 1; |
Adenilson Cavalcanti | 05e137d | 2022-09-09 18:14:22 +0000 | [diff] [blame] | 26 | int ZLIB_INTERNAL arm_cpu_enable_pmull = 1; |
Mark Mentovai | 8603eee | 2020-06-28 07:59:33 +0000 | [diff] [blame] | 27 | #else |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 28 | int ZLIB_INTERNAL arm_cpu_enable_crc32 = 0; |
Adenilson Cavalcanti | 0e6b3ca | 2022-05-23 07:17:30 +0000 | [diff] [blame] | 29 | int ZLIB_INTERNAL arm_cpu_enable_pmull = 0; |
Adenilson Cavalcanti | 05e137d | 2022-09-09 18:14:22 +0000 | [diff] [blame] | 30 | #endif |
Noel Gordon | f317558 | 2020-04-21 08:30:00 +0000 | [diff] [blame] | 31 | int ZLIB_INTERNAL x86_cpu_enable_sse2 = 0; |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 32 | int ZLIB_INTERNAL x86_cpu_enable_ssse3 = 0; |
| 33 | int ZLIB_INTERNAL x86_cpu_enable_simd = 0; |
Lei A Shi | b890619 | 2023-04-04 03:45:42 +0000 | [diff] [blame] | 34 | int ZLIB_INTERNAL x86_cpu_enable_avx512 = 0; |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 35 | |
Adenilson Cavalcanti | 24c07df | 2024-03-18 19:57:28 +0000 | [diff] [blame] | 36 | int ZLIB_INTERNAL riscv_cpu_enable_rvv = 0; |
| 37 | int ZLIB_INTERNAL riscv_cpu_enable_vclmul = 0; |
| 38 | |
Richard Townsend | c2eb8a7 | 2020-02-14 01:15:01 +0000 | [diff] [blame] | 39 | #ifndef CPU_NO_SIMD |
| 40 | |
Adenilson Cavalcanti | 24c07df | 2024-03-18 19:57:28 +0000 | [diff] [blame] | 41 | #if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || \ |
| 42 | defined(ARMV8_OS_FUCHSIA) || defined(ARMV8_OS_IOS) |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 43 | #include <pthread.h> |
| 44 | #endif |
| 45 | |
| 46 | #if defined(ARMV8_OS_ANDROID) |
| 47 | #include <cpu-features.h> |
| 48 | #elif defined(ARMV8_OS_LINUX) |
| 49 | #include <asm/hwcap.h> |
| 50 | #include <sys/auxv.h> |
| 51 | #elif defined(ARMV8_OS_FUCHSIA) |
| 52 | #include <zircon/features.h> |
| 53 | #include <zircon/syscalls.h> |
| 54 | #include <zircon/types.h> |
Hans Wennborg | 2a6432e | 2020-01-24 22:31:29 +0000 | [diff] [blame] | 55 | #elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS) |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 56 | #include <windows.h> |
Dave Tapuska | 1206f0d | 2023-07-11 20:11:07 +0000 | [diff] [blame] | 57 | #elif defined(ARMV8_OS_IOS) |
| 58 | #include <sys/sysctl.h> |
Hans Wennborg | 2a6432e | 2020-01-24 22:31:29 +0000 | [diff] [blame] | 59 | #elif !defined(_MSC_VER) |
Nico Weber | 51dd31c | 2020-01-24 20:43:07 +0000 | [diff] [blame] | 60 | #include <pthread.h> |
Hans Wennborg | 2a6432e | 2020-01-24 22:31:29 +0000 | [diff] [blame] | 61 | #else |
| 62 | #error cpu_features.c CPU feature detection in not defined for your platform |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 63 | #endif |
| 64 | |
Dave Tapuska | 1206f0d | 2023-07-11 20:11:07 +0000 | [diff] [blame] | 65 | #if !defined(CPU_NO_SIMD) && !defined(ARMV8_OS_MACOS) |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 66 | static void _cpu_check_features(void); |
| 67 | #endif |
| 68 | |
Adenilson Cavalcanti | 24c07df | 2024-03-18 19:57:28 +0000 | [diff] [blame] | 69 | #if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || \ |
| 70 | defined(ARMV8_OS_MACOS) || defined(ARMV8_OS_FUCHSIA) || \ |
| 71 | defined(X86_NOT_WINDOWS) || defined(ARMV8_OS_IOS) || \ |
| 72 | defined(RISCV_RVV) |
Nico Weber | 89bddfe | 2020-07-06 23:44:43 +0000 | [diff] [blame] | 73 | #if !defined(ARMV8_OS_MACOS) |
| 74 | // _cpu_check_features() doesn't need to do anything on mac/arm since all |
| 75 | // features are known at build time, so don't call it. |
| 76 | // Do provide cpu_check_features() (with a no-op implementation) so that we |
| 77 | // don't have to make all callers of it check for mac/arm. |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 78 | static pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT; |
Nico Weber | 89bddfe | 2020-07-06 23:44:43 +0000 | [diff] [blame] | 79 | #endif |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 80 | void ZLIB_INTERNAL cpu_check_features(void) |
| 81 | { |
Nico Weber | 89bddfe | 2020-07-06 23:44:43 +0000 | [diff] [blame] | 82 | #if !defined(ARMV8_OS_MACOS) |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 83 | pthread_once(&cpu_check_inited_once, _cpu_check_features); |
Nico Weber | 89bddfe | 2020-07-06 23:44:43 +0000 | [diff] [blame] | 84 | #endif |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 85 | } |
Hans Wennborg | 2a6432e | 2020-01-24 22:31:29 +0000 | [diff] [blame] | 86 | #elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS) |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 87 | static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT; |
| 88 | static BOOL CALLBACK _cpu_check_features_forwarder(PINIT_ONCE once, PVOID param, PVOID* context) |
| 89 | { |
| 90 | _cpu_check_features(); |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 91 | return TRUE; |
| 92 | } |
| 93 | void ZLIB_INTERNAL cpu_check_features(void) |
| 94 | { |
| 95 | InitOnceExecuteOnce(&cpu_check_inited_once, _cpu_check_features_forwarder, |
| 96 | NULL, NULL); |
| 97 | } |
| 98 | #endif |
| 99 | |
| 100 | #if (defined(__ARM_NEON__) || defined(__ARM_NEON)) |
Dave Tapuska | 1206f0d | 2023-07-11 20:11:07 +0000 | [diff] [blame] | 101 | #if !defined(ARMV8_OS_MACOS) |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 102 | /* |
| 103 | * See http://bit.ly/2CcoEsr for run-time detection of ARM features and also |
| 104 | * crbug.com/931275 for android_getCpuFeatures() use in the Android sandbox. |
| 105 | */ |
| 106 | static void _cpu_check_features(void) |
| 107 | { |
| 108 | #if defined(ARMV8_OS_ANDROID) && defined(__aarch64__) |
| 109 | uint64_t features = android_getCpuFeatures(); |
| 110 | arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM64_FEATURE_CRC32); |
| 111 | arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM64_FEATURE_PMULL); |
| 112 | #elif defined(ARMV8_OS_ANDROID) /* aarch32 */ |
| 113 | uint64_t features = android_getCpuFeatures(); |
| 114 | arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM_FEATURE_CRC32); |
| 115 | arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM_FEATURE_PMULL); |
| 116 | #elif defined(ARMV8_OS_LINUX) && defined(__aarch64__) |
| 117 | unsigned long features = getauxval(AT_HWCAP); |
| 118 | arm_cpu_enable_crc32 = !!(features & HWCAP_CRC32); |
| 119 | arm_cpu_enable_pmull = !!(features & HWCAP_PMULL); |
| 120 | #elif defined(ARMV8_OS_LINUX) && (defined(__ARM_NEON) || defined(__ARM_NEON__)) |
| 121 | /* Query HWCAP2 for ARMV8-A SoCs running in aarch32 mode */ |
| 122 | unsigned long features = getauxval(AT_HWCAP2); |
| 123 | arm_cpu_enable_crc32 = !!(features & HWCAP2_CRC32); |
| 124 | arm_cpu_enable_pmull = !!(features & HWCAP2_PMULL); |
| 125 | #elif defined(ARMV8_OS_FUCHSIA) |
| 126 | uint32_t features; |
| 127 | zx_status_t rc = zx_system_get_features(ZX_FEATURE_KIND_CPU, &features); |
| 128 | if (rc != ZX_OK || (features & ZX_ARM64_FEATURE_ISA_ASIMD) == 0) |
| 129 | return; /* Report nothing if ASIMD(NEON) is missing */ |
| 130 | arm_cpu_enable_crc32 = !!(features & ZX_ARM64_FEATURE_ISA_CRC32); |
| 131 | arm_cpu_enable_pmull = !!(features & ZX_ARM64_FEATURE_ISA_PMULL); |
| 132 | #elif defined(ARMV8_OS_WINDOWS) |
| 133 | arm_cpu_enable_crc32 = IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE); |
| 134 | arm_cpu_enable_pmull = IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE); |
Dave Tapuska | 1206f0d | 2023-07-11 20:11:07 +0000 | [diff] [blame] | 135 | #elif defined(ARMV8_OS_IOS) |
| 136 | // Determine what features are supported dynamically. This code is applicable to macOS |
| 137 | // as well if we wish to do that dynamically on that platform in the future. |
| 138 | // See https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics |
| 139 | int val = 0; |
| 140 | size_t len = sizeof(val); |
| 141 | arm_cpu_enable_crc32 = sysctlbyname("hw.optional.armv8_crc32", &val, &len, 0, 0) == 0 |
| 142 | && val != 0; |
| 143 | val = 0; |
| 144 | len = sizeof(val); |
| 145 | arm_cpu_enable_pmull = sysctlbyname("hw.optional.arm.FEAT_PMULL", &val, &len, 0, 0) == 0 |
| 146 | && val != 0; |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 147 | #endif |
| 148 | } |
| 149 | #endif |
| 150 | #elif defined(X86_NOT_WINDOWS) || defined(X86_WINDOWS) |
| 151 | /* |
| 152 | * iOS@x86 (i.e. emulator) is another special case where we disable |
| 153 | * SIMD optimizations. |
| 154 | */ |
| 155 | #ifndef CPU_NO_SIMD |
| 156 | /* On x86 we simply use a instruction to check the CPU features. |
| 157 | * (i.e. CPUID). |
| 158 | */ |
Lei A Shi | b890619 | 2023-04-04 03:45:42 +0000 | [diff] [blame] | 159 | #ifdef CRC32_SIMD_AVX512_PCLMUL |
| 160 | #include <immintrin.h> |
| 161 | #include <xsaveintrin.h> |
| 162 | #endif |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 163 | static void _cpu_check_features(void) |
| 164 | { |
| 165 | int x86_cpu_has_sse2; |
| 166 | int x86_cpu_has_ssse3; |
| 167 | int x86_cpu_has_sse42; |
| 168 | int x86_cpu_has_pclmulqdq; |
| 169 | int abcd[4]; |
Noel Gordon | f317558 | 2020-04-21 08:30:00 +0000 | [diff] [blame] | 170 | |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 171 | #ifdef _MSC_VER |
| 172 | __cpuid(abcd, 1); |
| 173 | #else |
| 174 | __cpuid(1, abcd[0], abcd[1], abcd[2], abcd[3]); |
| 175 | #endif |
Noel Gordon | f317558 | 2020-04-21 08:30:00 +0000 | [diff] [blame] | 176 | |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 177 | x86_cpu_has_sse2 = abcd[3] & 0x4000000; |
| 178 | x86_cpu_has_ssse3 = abcd[2] & 0x000200; |
| 179 | x86_cpu_has_sse42 = abcd[2] & 0x100000; |
| 180 | x86_cpu_has_pclmulqdq = abcd[2] & 0x2; |
| 181 | |
Noel Gordon | f317558 | 2020-04-21 08:30:00 +0000 | [diff] [blame] | 182 | x86_cpu_enable_sse2 = x86_cpu_has_sse2; |
| 183 | |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 184 | x86_cpu_enable_ssse3 = x86_cpu_has_ssse3; |
| 185 | |
| 186 | x86_cpu_enable_simd = x86_cpu_has_sse2 && |
| 187 | x86_cpu_has_sse42 && |
| 188 | x86_cpu_has_pclmulqdq; |
Lei A Shi | b890619 | 2023-04-04 03:45:42 +0000 | [diff] [blame] | 189 | |
| 190 | #ifdef CRC32_SIMD_AVX512_PCLMUL |
| 191 | x86_cpu_enable_avx512 = _xgetbv(0) & 0x00000040; |
| 192 | #endif |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 193 | } |
Adenilson Cavalcanti | 24c07df | 2024-03-18 19:57:28 +0000 | [diff] [blame] | 194 | #endif // x86 & NO_SIMD |
| 195 | |
| 196 | #elif defined(RISCV_RVV) |
| 197 | #include <sys/auxv.h> |
| 198 | |
| 199 | #ifndef ZLIB_HWCAP_RVV |
| 200 | #define ZLIB_HWCAP_RVV (1 << ('v' - 'a')) |
Adenilson Cavalcanti | 5de00af | 2020-01-08 22:12:31 +0000 | [diff] [blame] | 201 | #endif |
Adenilson Cavalcanti | 24c07df | 2024-03-18 19:57:28 +0000 | [diff] [blame] | 202 | |
| 203 | /* TODO(cavalcantii) |
| 204 | * - add support for Android@RISCV i.e. __riscv_hwprobe(). |
| 205 | * - detect vclmul (crypto extensions). |
| 206 | */ |
| 207 | static void _cpu_check_features(void) |
| 208 | { |
| 209 | unsigned long features = getauxval(AT_HWCAP); |
| 210 | riscv_cpu_enable_rvv = !!(features & ZLIB_HWCAP_RVV); |
| 211 | } |
| 212 | #endif // ARM | x86 | RISCV |
| 213 | #endif // NO SIMD CPU |