Skip to content

Commit 00a784f

Browse files
authored
[libc][math] Add do not optimize macros and guard some double-double functions. (#243203)
Guard some double-double computations against `-(a * b) -> a * (-b)` optimizations from compilers.
1 parent af66e10 commit 00a784f

9 files changed

Lines changed: 276 additions & 0 deletions

File tree

‎libc/src/__support/FPUtil/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,7 @@ add_header_library(
192192
DEPENDS
193193
libc.src.__support.common
194194
libc.src.__support.number_pair
195+
libc.src.__support.macros.optimization
195196
libc.src.__support.macros.properties.cpu_features
196197
.multiply_add
197198
)

‎libc/src/__support/FPUtil/double_double.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "multiply_add.h"
1313
#include "src/__support/common.h"
1414
#include "src/__support/macros/config.h"
15+
#include "src/__support/macros/optimization.h" // LIBC_DO_NOT_OPTIMIZE
1516
#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
1617
#include "src/__support/number_pair.h"
1718

@@ -173,6 +174,10 @@ LIBC_INLINE LIBC_CONSTEXPR NumberPair<T> exact_mult(T a, T b) {
173174

174175
if constexpr (TargetHasFmaInstruction<T>::VALUE) {
175176
r.hi = a * b;
177+
// Prevent `-r.hi` below from being rewritten into `a * (-b)` when `a * b`
178+
// is duplicated (e.g. by SLP vectorization), which breaks directed
179+
// rounding modes (`RU(a * (-b)) != -RU(a * b)`).
180+
LIBC_DO_NOT_OPTIMIZE_ROUNDING(r.hi);
176181
r.lo = fputil::multiply_add(a, b, -r.hi);
177182
} else {
178183
// Dekker's Product.
@@ -231,6 +236,7 @@ LIBC_INLINE NumberPair<T> div(const NumberPair<T> &a, const NumberPair<T> &b) {
231236
NumberPair<T> r;
232237
T q = T(1) / b.hi;
233238
r.hi = a.hi * q;
239+
LIBC_DO_NOT_OPTIMIZE_ROUNDING(r.hi);
234240

235241
#ifdef LIBC_TARGET_CPU_HAS_FMA
236242
T e_hi = fputil::multiply_add(b.hi, -r.hi, a.hi);

‎libc/src/__support/macros/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,11 @@ add_header_library(
2020
HDRS
2121
optimization.h
2222
DEPENDS
23+
.attributes
24+
.config
25+
libc.src.__support.macros.properties.architectures
2326
libc.src.__support.macros.properties.compiler
27+
libc.src.__support.macros.properties.cpu_features
2428
)
2529

2630
add_header_library(

‎libc/src/__support/macros/optimization.h‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212

1313
#include "src/__support/macros/attributes.h" // LIBC_INLINE
1414
#include "src/__support/macros/config.h"
15+
#include "src/__support/macros/properties/architectures.h"
1516
#include "src/__support/macros/properties/compiler.h" // LIBC_COMPILER_IS_CLANG
17+
#include "src/__support/macros/properties/cpu_features.h"
1618

1719
// We use a template to implement likely/unlikely to make sure that we don't
1820
// accidentally pass an integer.
@@ -27,8 +29,71 @@ template <>
2729
LIBC_ALWAYS_INLINE constexpr bool expects_bool_condition<bool>(bool value) {
2830
return value;
2931
}
32+
33+
template <typename T> void do_not_optimize_impl(T &) = delete;
34+
35+
// Zero-cost SSA value barrier for float and double: keeps `val` in its native
36+
// register class without spilling to memory or moving across GPR/FPR banks,
37+
// while preventing the optimizer from looking through or duplicating `val`.
38+
// Kept in non-constexpr helpers because C++17 rejects inline `asm` statements
39+
// directly inside `constexpr` functions (`-Wc++20-extensions`). Skipped on
40+
// `cl.exe` (`LIBC_COMPILER_IS_MSVC`, which excludes `clang-cl`) because MSVC
41+
// does not support GNU extended `asm` register constraints.
42+
LIBC_ALWAYS_INLINE void do_not_optimize_impl(LIBC_UNUSED float &val) {
43+
#if !defined(LIBC_COMPILER_IS_MSVC)
44+
#if defined(LIBC_TARGET_CPU_HAS_ARM_FPU_FLOAT)
45+
asm("" : "+w"(val));
46+
#elif defined(LIBC_TARGET_ARCH_IS_X86) && defined(LIBC_TARGET_CPU_HAS_SSE2)
47+
asm("" : "+x"(val));
48+
#elif defined(LIBC_TARGET_ARCH_IS_X86)
49+
asm("" : "+t"(val));
50+
#elif defined(LIBC_TARGET_CPU_HAS_RISCV_FPU_FLOAT)
51+
asm("" : "+f"(val));
52+
#elif defined(LIBC_TARGET_ARCH_IS_AMDGPU)
53+
asm("" : "+v"(val));
54+
#elif defined(LIBC_TARGET_ARCH_IS_NVPTX)
55+
asm("" : "+f"(val));
56+
#elif defined(LIBC_TARGET_ARCH_IS_ARM) || \
57+
defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) || \
58+
defined(LIBC_TARGET_ARCH_IS_WASM) || defined(LIBC_TARGET_ARCH_IS_HEXAGON)
59+
asm("" : "+r"(val));
60+
#endif
61+
#endif // !LIBC_COMPILER_IS_MSVC
62+
}
63+
64+
LIBC_ALWAYS_INLINE void do_not_optimize_impl(LIBC_UNUSED double &val) {
65+
#if !defined(LIBC_COMPILER_IS_MSVC)
66+
#if defined(LIBC_TARGET_CPU_HAS_ARM_FPU_DOUBLE)
67+
asm("" : "+w"(val));
68+
#elif defined(LIBC_TARGET_ARCH_IS_X86) && defined(LIBC_TARGET_CPU_HAS_SSE2)
69+
asm("" : "+x"(val));
70+
#elif defined(LIBC_TARGET_ARCH_IS_X86)
71+
asm("" : "+t"(val));
72+
#elif defined(LIBC_TARGET_CPU_HAS_RISCV_FPU_DOUBLE)
73+
asm("" : "+f"(val));
74+
#elif defined(LIBC_TARGET_ARCH_IS_AMDGPU)
75+
asm("" : "+v"(val));
76+
#elif defined(LIBC_TARGET_ARCH_IS_NVPTX)
77+
asm("" : "+d"(val));
78+
#elif defined(LIBC_TARGET_ARCH_IS_RISCV64) || defined(LIBC_TARGET_ARCH_IS_WASM)
79+
asm("" : "+r"(val));
80+
#endif
81+
#endif // !LIBC_COMPILER_IS_MSVC
82+
}
83+
84+
template <typename T>
85+
LIBC_ALWAYS_INLINE LIBC_CONSTEXPR_DEFAULT void do_not_optimize(T &val) {
86+
#if LIBC_HAS_BUILTIN_IS_CONSTANT_EVALUATED
87+
if (!__builtin_is_constant_evaluated())
88+
#endif
89+
do_not_optimize_impl(val);
90+
}
91+
3092
} // namespace details
3193
} // namespace LIBC_NAMESPACE_DECL
94+
95+
#define LIBC_DO_NOT_OPTIMIZE(var) LIBC_NAMESPACE::details::do_not_optimize(var)
96+
3297
#define LIBC_LIKELY(x) \
3398
(__builtin_expect(LIBC_NAMESPACE::details::expects_bool_condition((x)), true))
3499
#define LIBC_UNLIKELY(x) \
@@ -95,6 +160,9 @@ LIBC_ALWAYS_INLINE constexpr bool expects_bool_condition<bool>(bool value) {
95160

96161
#if ((LIBC_MATH) & LIBC_MATH_ASSUME_ROUND_NEAREST_ONLY)
97162
#define LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY
163+
#define LIBC_DO_NOT_OPTIMIZE_ROUNDING(var) ((void)(var))
164+
#else
165+
#define LIBC_DO_NOT_OPTIMIZE_ROUNDING(var) LIBC_DO_NOT_OPTIMIZE(var)
98166
#endif
99167

100168
// Helper macros to compare math implementation namespace tokens in #if

‎libc/test/src/__support/FPUtil/CMakeLists.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,18 @@ add_fp_unittest(
7373
libc.src.__support.FPUtil.rounding_mode
7474
)
7575

76+
add_fp_unittest(
77+
double_double_test
78+
SUITE
79+
libc-fputil-tests
80+
SRCS
81+
double_double_test.cpp
82+
DEPENDS
83+
libc.src.__support.FPUtil.double_double
84+
libc.src.__support.macros.config
85+
libc.src.__support.macros.optimization
86+
)
87+
7688
add_fp_unittest(
7789
float128_test
7890
SUITE
Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
///
9+
/// \file
10+
/// Unittests for LIBC_DO_NOT_OPTIMIZE and double-double utilities.
11+
///
12+
//===----------------------------------------------------------------------===//
13+
14+
#include "src/__support/FPUtil/double_double.h"
15+
#include "src/__support/macros/config.h"
16+
#include "src/__support/macros/optimization.h"
17+
#include "test/UnitTest/FEnvSafeTest.h"
18+
#include "test/UnitTest/FPMatcher.h"
19+
#include "test/UnitTest/RoundingModeUtils.h"
20+
#include "test/UnitTest/Test.h"
21+
22+
using LIBC_NAMESPACE::fputil::DoubleDouble;
23+
using LIBC_NAMESPACE::fputil::testing::ForceRoundingMode;
24+
using LIBC_NAMESPACE::fputil::testing::RoundingMode;
25+
26+
namespace {
27+
28+
#if LIBC_HAS_CONSTANT_EVALUATION
29+
constexpr float constexpr_barrier_check_f(float x) {
30+
LIBC_DO_NOT_OPTIMIZE(x);
31+
LIBC_DO_NOT_OPTIMIZE_ROUNDING(x);
32+
return x + 1.0f;
33+
}
34+
constexpr double constexpr_barrier_check_d(double x) {
35+
LIBC_DO_NOT_OPTIMIZE(x);
36+
LIBC_DO_NOT_OPTIMIZE_ROUNDING(x);
37+
return x + 1.0;
38+
}
39+
static_assert(constexpr_barrier_check_f(2.5f) == 3.5f);
40+
static_assert(constexpr_barrier_check_d(2.5) == 3.5);
41+
#endif
42+
43+
} // namespace
44+
45+
class LlvmLibcDoubleDoubleTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
46+
};
47+
48+
TEST_F(LlvmLibcDoubleDoubleTest, DoNotOptimizeFloatAndDouble) {
49+
float f = 1.5f;
50+
LIBC_DO_NOT_OPTIMIZE(f);
51+
LIBC_DO_NOT_OPTIMIZE_ROUNDING(f);
52+
EXPECT_FP_EQ(f, 1.5f);
53+
54+
double d = -2.25;
55+
LIBC_DO_NOT_OPTIMIZE(d);
56+
LIBC_DO_NOT_OPTIMIZE_ROUNDING(d);
57+
EXPECT_FP_EQ(d, -2.25);
58+
}
59+
60+
#if !defined(LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY) && \
61+
!defined(LIBC_COMPILER_IS_MSVC)
62+
TEST_F(LlvmLibcDoubleDoubleTest, DoNotOptimizeBlocksConstantFolding) {
63+
{
64+
ForceRoundingMode r(RoundingMode::Upward);
65+
if (r.success) {
66+
float one_f = 1.0f;
67+
LIBC_DO_NOT_OPTIMIZE(one_f);
68+
EXPECT_FP_EQ(one_f + 0x1.0p-50f, 0x1.000002p0f);
69+
70+
double one_d = 1.0;
71+
LIBC_DO_NOT_OPTIMIZE_ROUNDING(one_d);
72+
EXPECT_FP_EQ(one_d + 0x1.0p-100, 0x1.0000000000001p0);
73+
}
74+
}
75+
{
76+
ForceRoundingMode r(RoundingMode::Downward);
77+
if (r.success) {
78+
float neg_one_f = -1.0f;
79+
LIBC_DO_NOT_OPTIMIZE(neg_one_f);
80+
EXPECT_FP_EQ(neg_one_f - 0x1.0p-50f, -0x1.000002p0f);
81+
82+
double neg_one_d = -1.0;
83+
LIBC_DO_NOT_OPTIMIZE_ROUNDING(neg_one_d);
84+
EXPECT_FP_EQ(neg_one_d - 0x1.0p-100, -0x1.0000000000001p0);
85+
}
86+
}
87+
}
88+
#endif
89+
90+
TEST_F(LlvmLibcDoubleDoubleTest, ExactMultDirectedRounding) {
91+
if constexpr (LIBC_NAMESPACE::fputil::TargetHasFmaInstruction<
92+
double>::VALUE) {
93+
// Exact product: (1 + 2^-52)^2 = (1 + 2^-51) + 2^-104
94+
// = (1 + 3*2^-52) + (-2^-52 + 2^-104).
95+
auto check_square = [](RoundingMode mode, double expected_hi,
96+
double expected_lo) {
97+
ForceRoundingMode r(mode);
98+
if (!r.success)
99+
return;
100+
double a = 1.0 + 0x1.0p-52;
101+
LIBC_DO_NOT_OPTIMIZE(a);
102+
DoubleDouble prod = LIBC_NAMESPACE::fputil::exact_mult(a, a);
103+
EXPECT_FP_EQ(prod.hi, expected_hi);
104+
EXPECT_FP_EQ(prod.lo, expected_lo);
105+
};
106+
107+
check_square(RoundingMode::Nearest, 1.0 + 0x1.0p-51, 0x1.0p-104);
108+
#ifndef LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY
109+
check_square(RoundingMode::Upward, 1.0 + 0x1.8p-51,
110+
-0x1.0p-52 + 0x1.0p-104);
111+
check_square(RoundingMode::Downward, 1.0 + 0x1.0p-51, 0x1.0p-104);
112+
check_square(RoundingMode::TowardZero, 1.0 + 0x1.0p-51, 0x1.0p-104);
113+
#endif
114+
115+
// Four-way exact_mult + exact_add<false> cancellation pattern (as in
116+
// sincos, b/586440175):
117+
// p0 = (1 + 2^-52) * (1 + 2^-52) = 1 + 2^-51 + 2^-104
118+
// p1 = (1 + 2^-51) * -(1 + 2^-51) = -1 - 2^-50 - 2^-102
119+
// p0 + p1 = -(2^-51 + 1.5 * 2^-103) (exact midpoint in ulp(2^-51))
120+
// p2 = (1 + 2^-50) * (1 + 2^-50) = 1 + 2^-49 + 2^-100
121+
// p3 = (1 + 2^-49) * -(1 + 2^-49) = -1 - 2^-48 - 2^-98
122+
// p2 + p3 = -(2^-49 + 3 * 2^-100) (exact binary64:
123+
// -0x1.0000000000006p-49)
124+
auto check_four_way = [](RoundingMode mode, double expected_s0,
125+
double expected_s1) {
126+
ForceRoundingMode r(mode);
127+
if (!r.success)
128+
return;
129+
130+
double a0 = 1.0 + 0x1.0p-52;
131+
double b0 = 1.0 + 0x1.0p-52;
132+
double a1 = 1.0 + 0x1.0p-51;
133+
double b1 = -(1.0 + 0x1.0p-51);
134+
double a2 = 1.0 + 0x1.0p-50;
135+
double b2 = 1.0 + 0x1.0p-50;
136+
double a3 = 1.0 + 0x1.0p-49;
137+
double b3 = -(1.0 + 0x1.0p-49);
138+
LIBC_DO_NOT_OPTIMIZE(a0);
139+
LIBC_DO_NOT_OPTIMIZE(b0);
140+
LIBC_DO_NOT_OPTIMIZE(a1);
141+
LIBC_DO_NOT_OPTIMIZE(b1);
142+
LIBC_DO_NOT_OPTIMIZE(a2);
143+
LIBC_DO_NOT_OPTIMIZE(b2);
144+
LIBC_DO_NOT_OPTIMIZE(a3);
145+
LIBC_DO_NOT_OPTIMIZE(b3);
146+
147+
DoubleDouble p0 = LIBC_NAMESPACE::fputil::exact_mult(a0, b0);
148+
DoubleDouble p1 = LIBC_NAMESPACE::fputil::exact_mult(a1, b1);
149+
DoubleDouble p2 = LIBC_NAMESPACE::fputil::exact_mult(a2, b2);
150+
DoubleDouble p3 = LIBC_NAMESPACE::fputil::exact_mult(a3, b3);
151+
152+
DoubleDouble s0 = LIBC_NAMESPACE::fputil::exact_add<false>(p0.hi, p1.hi);
153+
DoubleDouble s1 = LIBC_NAMESPACE::fputil::exact_add<false>(p2.hi, p3.hi);
154+
s0.lo += p0.lo + p1.lo;
155+
s1.lo += p2.lo + p3.lo;
156+
157+
EXPECT_FP_EQ(s0.hi + s0.lo, expected_s0);
158+
EXPECT_FP_EQ(s1.hi + s1.lo, expected_s1);
159+
};
160+
161+
check_four_way(RoundingMode::Nearest, -0x1.0000000000002p-51,
162+
-0x1.0000000000006p-49);
163+
#ifndef LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY
164+
check_four_way(RoundingMode::Upward, -0x1.0000000000001p-51,
165+
-0x1.0000000000006p-49);
166+
check_four_way(RoundingMode::Downward, -0x1.0000000000002p-51,
167+
-0x1.0000000000006p-49);
168+
check_four_way(RoundingMode::TowardZero, -0x1.0000000000001p-51,
169+
-0x1.0000000000006p-49);
170+
#endif
171+
}
172+
}

‎utils/bazel/llvm-project-overlay/libc/src/__support/FPUtil/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,7 @@ libc_support_library(
434434
"//libc/src/__support:common",
435435
"//libc/src/__support:number_pair",
436436
"//libc/src/__support/macros:config",
437+
"//libc/src/__support/macros:optimization",
437438
"//libc/src/__support/macros/properties:cpu_features",
438439
],
439440
)

‎utils/bazel/llvm-project-overlay/libc/src/__support/macros/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,9 @@ libc_support_library(
5858
deps = [
5959
":attributes",
6060
":config",
61+
"//libc/src/__support/macros/properties:architectures",
6162
"//libc/src/__support/macros/properties:compiler",
63+
"//libc/src/__support/macros/properties:cpu_features",
6264
],
6365
)
6466

‎utils/bazel/llvm-project-overlay/libc/test/src/__support/FPUtil/BUILD.bazel‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,16 @@ package(default_visibility = ["//visibility:public"])
1010

1111
licenses(["notice"])
1212

13+
libc_test(
14+
name = "double_double_test",
15+
srcs = ["double_double_test.cpp"],
16+
deps = [
17+
"//libc/src/__support/FPUtil:double_double",
18+
"//libc/src/__support/macros:optimization",
19+
"//libc/test/UnitTest:fp_test_helpers",
20+
],
21+
)
22+
1323
libc_test(
1424
name = "fpbits_test",
1525
srcs = ["fpbits_test.cpp"],

0 commit comments

Comments
 (0)