diff --git a/benchmarks/string_benchmark.cpp b/benchmarks/string_benchmark.cpp index 536e253fd..6d7c251ad 100644 --- a/benchmarks/string_benchmark.cpp +++ b/benchmarks/string_benchmark.cpp @@ -110,3 +110,129 @@ static void BM_string_strlen(int iters, int nbytes) { delete[] s; } BENCHMARK(BM_string_strlen)->AT_COMMON_SIZES; + +static void BM_string_strchr(int iters, int nbytes) { + StopBenchmarkTiming(); + char* s = new char[nbytes]; + memset(s, 'x', nbytes); + s[nbytes - 1] = 0; + StartBenchmarkTiming(); + + volatile char* c = new char[1]; + volatile int d __attribute__((unused)) = 0; + for (int i = 0; i < iters; ++i) { + c = strchr(s, 'y'); + if (c != NULL) + d += (int)c[0]; + } + + StopBenchmarkTiming(); + SetBenchmarkBytesProcessed(int64_t(iters) * int64_t(nbytes)); + delete[] s; +} +BENCHMARK(BM_string_strchr)->AT_COMMON_SIZES; + +static void BM_string_strrchr(int iters, int nbytes) { + StopBenchmarkTiming(); + char* s = new char[nbytes]; + memset(s, 'x', nbytes); + s[nbytes - 1] = 0; + StartBenchmarkTiming(); + + volatile char* c = new char[1]; + volatile int d __attribute__((unused)) = 0; + for (int i = 0; i < iters; ++i) { + c = strrchr(s, 'y'); + if (c != NULL) + d += (int)c[0]; + } + + StopBenchmarkTiming(); + SetBenchmarkBytesProcessed(int64_t(iters) * int64_t(nbytes)); + delete[] s; +} +BENCHMARK(BM_string_strrchr)->AT_COMMON_SIZES; + +static void BM_string_strcmp(int iters, int nbytes) { + StopBenchmarkTiming(); + char* src = new char[nbytes]; char* dst = new char[nbytes]; + memset(src, 'x', nbytes); + memset(dst, 'x', nbytes); + src[nbytes - 1] = 0; + dst[nbytes - 1] = 0; + StartBenchmarkTiming(); + + volatile int c __attribute__((unused)) = 0; + for (int i = 0; i < iters; ++i) { + c += strcmp(dst, src); + } + + StopBenchmarkTiming(); + SetBenchmarkBytesProcessed(int64_t(iters) * int64_t(nbytes)); + delete[] src; + delete[] dst; +} +BENCHMARK(BM_string_strcmp)->AT_COMMON_SIZES; + +static void BM_string_strncmp(int iters, int nbytes) { + StopBenchmarkTiming(); + char* src = new char[nbytes]; char* dst = new char[nbytes]; + memset(src, 'x', nbytes); + memset(dst, 'x', nbytes); + src[nbytes - 1] = 0; + dst[nbytes - 1] = 0; + StartBenchmarkTiming(); + + volatile int c __attribute__((unused)) = 0; + for (int i = 0; i < iters; ++i) { + c += strncmp(dst, src, nbytes - 1); + } + + StopBenchmarkTiming(); + SetBenchmarkBytesProcessed(int64_t(iters) * int64_t(nbytes)); + delete[] src; + delete[] dst; +} +BENCHMARK(BM_string_strncmp)->AT_COMMON_SIZES; + +static void BM_string_memchr(int iters, int nbytes) { + StopBenchmarkTiming(); + char* s = new char[nbytes]; + memset(s, 'x', nbytes); + s[nbytes - 1] = 0; + StartBenchmarkTiming(); + + volatile void* c __attribute__((unused)) = NULL; + volatile int d __attribute__((unused)) = 0; + for (int i = 0; i < iters; ++i) { + c = memchr(s, 'y', nbytes - 1); + if (c != NULL) + d += (int)c; + } + + StopBenchmarkTiming(); + SetBenchmarkBytesProcessed(int64_t(iters) * int64_t(nbytes)); + delete[] s; +} +BENCHMARK(BM_string_memchr)->AT_COMMON_SIZES; + +static void BM_string_memrchr(int iters, int nbytes) { + StopBenchmarkTiming(); + char* s = new char[nbytes]; + memset(s, 'x', nbytes); + s[nbytes - 1] = 0; + StartBenchmarkTiming(); + + volatile void* c __attribute__((unused)) = NULL; + volatile int d __attribute__((unused)) = 0; + for (int i = 0; i < iters; ++i) { + c = memrchr(s, 'y', nbytes - 1); + if (c != NULL) + d += (int)c; + } + + StopBenchmarkTiming(); + SetBenchmarkBytesProcessed(int64_t(iters) * int64_t(nbytes)); + delete[] s; +} +BENCHMARK(BM_string_memrchr)->AT_COMMON_SIZES; diff --git a/libc/Android.mk b/libc/Android.mk index 30050922b..b59f66993 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -65,6 +65,10 @@ libc_common_src_files := \ stdio/snprintf.c\ stdio/sprintf.c \ +ifeq ($(TARGET_NEEDS_BIONIC_MD5),true) +libc_common_src_files += bionic/md5.c +endif + # Fortify implementations of libc functions. libc_common_src_files += \ bionic/__FD_chk.cpp \ @@ -85,6 +89,10 @@ libc_common_src_files += \ bionic/__vsnprintf_chk.cpp \ bionic/__vsprintf_chk.cpp \ +ifeq ($(TARGET_NEEDS_BIONIC_MD5),true) + libc_common_src_files += bionic/md5.c +endif + libc_bionic_src_files := \ bionic/abort.cpp \ bionic/accept.cpp \ @@ -535,6 +543,10 @@ ifneq ($(BOARD_MALLOC_ALIGNMENT),) libc_common_cflags += -DMALLOC_ALIGNMENT=$(BOARD_MALLOC_ALIGNMENT) endif +ifeq ($(BOARD_USES_LEGACY_MMAP),true) + libc_common_cflags += -DLEGACY_MMAP +endif + # Define ANDROID_SMP appropriately. ifeq ($(TARGET_CPU_SMP),true) libc_common_cflags += -DANDROID_SMP=1 @@ -813,11 +825,6 @@ LOCAL_SRC_FILES := $(libc_bionic_src_files) LOCAL_CFLAGS := $(libc_common_cflags) \ -Wframe-larger-than=2048 \ -ifeq ($(TARGET_ARCH),x86_64) - # Clang assembler has problem with ssse3-strcmp-slm.S, http://b/17302991 - LOCAL_CLANG_ASFLAGS += -no-integrated-as -endif - LOCAL_CONLYFLAGS := $(libc_common_conlyflags) LOCAL_CPPFLAGS := $(libc_common_cppflags) LOCAL_C_INCLUDES := $(libc_common_c_includes) @@ -1069,6 +1076,9 @@ LOCAL_SRC_FILES_arm += \ arch-arm/bionic/atexit_legacy.c \ arch-common/bionic/crtend_so.S +# Allow devices to provide additional symbols +LOCAL_WHOLE_STATIC_LIBRARIES += $(BOARD_PROVIDES_ADDITIONAL_BIONIC_STATIC_LIBS) + include $(BUILD_SHARED_LIBRARY) @@ -1079,6 +1089,7 @@ include $(BUILD_SHARED_LIBRARY) # the checks are available only in emulator and are implemeted in # libc_malloc_qemu_instrumented.so. ifneq ($(TARGET_BUILD_VARIANT),user) +ifneq ($(TARGET_BUILD_VARIANT),userdebug) # ======================================================== # libc_malloc_debug_leak.so @@ -1115,7 +1126,7 @@ LOCAL_STATIC_LIBRARIES_arm := libc++ LOCAL_ALLOW_UNDEFINED_SYMBOLS := true # Don't install on release build -LOCAL_MODULE_TAGS := eng debug +LOCAL_MODULE_TAGS := eng $(eval $(call patch-up-arch-specific-flags,LOCAL_CFLAGS,libc_common_cflags)) include $(BUILD_SHARED_LIBRARY) @@ -1147,11 +1158,12 @@ LOCAL_SHARED_LIBRARIES := libc libdl LOCAL_SYSTEM_SHARED_LIBRARIES := # Don't install on release build -LOCAL_MODULE_TAGS := eng debug +LOCAL_MODULE_TAGS := eng $(eval $(call patch-up-arch-specific-flags,LOCAL_CFLAGS,libc_common_cflags)) include $(BUILD_SHARED_LIBRARY) +endif #!userdebug endif #!user # ======================================================== diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index cca4ed020..707e6479f 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -23,11 +23,8 @@ libc_openbsd_src_files_arm += \ # libc_bionic_src_files_arm += \ - bionic/memchr.c \ bionic/memrchr.c \ - bionic/strchr.cpp \ bionic/strnlen.c \ - bionic/strrchr.cpp \ libc_freebsd_src_files_arm += \ upstream-freebsd/lib/libc/string/wcscat.c \ @@ -40,7 +37,6 @@ libc_freebsd_src_files_arm += \ upstream-freebsd/lib/libc/string/wmemmove.c \ libc_openbsd_src_files_arm += \ - upstream-openbsd/lib/libc/string/bcopy.c \ upstream-openbsd/lib/libc/string/stpncpy.c \ upstream-openbsd/lib/libc/string/strlcat.c \ upstream-openbsd/lib/libc/string/strlcpy.c \ @@ -64,6 +60,8 @@ libc_bionic_src_files_arm += \ arch-arm/bionic/setjmp.S \ arch-arm/bionic/sigsetjmp.S \ arch-arm/bionic/syscall.S \ + arch-arm/bionic/strchr.S \ + arch-arm/bionic/strrchr.S \ libc_arch_static_src_files_arm := arch-arm/bionic/exidx_static.c libc_arch_dynamic_src_files_arm := arch-arm/bionic/exidx_dynamic.c @@ -72,9 +70,20 @@ libc_arch_dynamic_src_files_arm := arch-arm/bionic/exidx_dynamic.c ifeq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),) $(warning TARGET_$(my_2nd_arch_prefix)ARCH is arm, but TARGET_$(my_2nd_arch_prefix)CPU_VARIANT is not defined) endif + +ifeq ($(filter cortex-a9 cortex-a53 denver krait scorpion, $(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT))),) +libc_bionic_src_files_arm += \ + upstream-openbsd/lib/libc/string/stpcpy.c +endif + +ifeq ($(filter cortex-a53 krait scorpion, $(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT))),) +libc_bionic_src_files_arm += \ + upstream-openbsd/lib/libc/string/bcopy.c +endif + cpu_variant_mk := $(LOCAL_PATH)/arch-arm/$(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)/$(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT).mk ifeq ($(wildcard $(cpu_variant_mk)),) -$(error "TARGET_$(my_2nd_arch_prefix)CPU_VARIANT not set or set to an unknown value. Possible values are cortex-a7, cortex-a8, cortex-a9, cortex-a15, krait, denver. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.") +$(error "TARGET_$(my_2nd_arch_prefix)CPU_VARIANT not set or set to an unknown value. Possible values are cortex-a7, cortex-a8, cortex-a9, cortex-a15, krait, scorpion, denver. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.") endif include $(cpu_variant_mk) libc_common_additional_dependencies += $(cpu_variant_mk) diff --git a/libc/arch-arm/bionic/libgcc_compat.c b/libc/arch-arm/bionic/libgcc_compat.c index 10faf2d1b..88ed81389 100644 --- a/libc/arch-arm/bionic/libgcc_compat.c +++ b/libc/arch-arm/bionic/libgcc_compat.c @@ -7,6 +7,7 @@ extern char __aeabi_cdcmple; extern char __aeabi_cdrcmple; extern char __aeabi_d2f; extern char __aeabi_d2iz; +extern char __aeabi_d2uiz; extern char __aeabi_dadd; extern char __aeabi_dcmpeq; extern char __aeabi_dcmpge; @@ -22,6 +23,7 @@ extern char __aeabi_f2d; extern char __aeabi_f2iz; extern char __aeabi_f2uiz; extern char __aeabi_fadd; +extern char __aeabi_fcmpgt; extern char __aeabi_fcmpun; extern char __aeabi_fdiv; extern char __aeabi_fmul; @@ -86,6 +88,7 @@ void* __bionic_libgcc_compat_symbols[] = { &__aeabi_cdrcmple, &__aeabi_d2f, &__aeabi_d2iz, + &__aeabi_d2uiz, &__aeabi_dadd, &__aeabi_dcmpeq, &__aeabi_dcmpge, @@ -101,6 +104,7 @@ void* __bionic_libgcc_compat_symbols[] = { &__aeabi_f2iz, &__aeabi_f2uiz, &__aeabi_fadd, + &__aeabi_fcmpgt, &__aeabi_fcmpun, &__aeabi_fdiv, &__aeabi_fmul, diff --git a/libc/arch-arm/bionic/memcmp.S b/libc/arch-arm/bionic/memcmp.S index 70a2a58ce..886d89923 100644 --- a/libc/arch-arm/bionic/memcmp.S +++ b/libc/arch-arm/bionic/memcmp.S @@ -40,6 +40,8 @@ * Optimized memcmp() for Cortex-A9. */ +.syntax unified + ENTRY(memcmp) pld [r0, #(CACHE_LINE_SIZE * 0)] pld [r0, #(CACHE_LINE_SIZE * 1)] @@ -61,7 +63,7 @@ ENTRY(memcmp) * Neon optimization * Comparing 32 bytes at a time */ -#if defined(__ARM_NEON__) && defined(NEON_UNALIGNED_ACCESS) +#if defined(__ARM_NEON__) && defined(__ARM_FEATURE_UNALIGNED) subs r2, r2, #32 blo 3f @@ -161,25 +163,25 @@ ENTRY(memcmp) eors r0, r0, ip ldreq r0, [r4], #4 ldreq ip, [r1, #4]! - eoreqs r0, r0, lr + eorseq r0, r0, lr ldreq r0, [r4], #4 ldreq lr, [r1, #4]! - eoreqs r0, r0, ip + eorseq r0, r0, ip ldreq r0, [r4], #4 ldreq ip, [r1, #4]! - eoreqs r0, r0, lr + eorseq r0, r0, lr ldreq r0, [r4], #4 ldreq lr, [r1, #4]! - eoreqs r0, r0, ip + eorseq r0, r0, ip ldreq r0, [r4], #4 ldreq ip, [r1, #4]! - eoreqs r0, r0, lr + eorseq r0, r0, lr ldreq r0, [r4], #4 ldreq lr, [r1, #4]! - eoreqs r0, r0, ip + eorseq r0, r0, ip ldreq r0, [r4], #4 ldreq ip, [r1, #4]! - eoreqs r0, r0, lr + eorseq r0, r0, lr bne 2f subs r2, r2, #32 bhs 0b @@ -219,8 +221,7 @@ ENTRY(memcmp) bne 8b 9: /* restore registers and return */ - ldmfd sp!, {r4, lr} - bx lr + ldmfd sp!, {r4, pc} 10: /* process less than 12 bytes */ cmp r2, #0 @@ -263,17 +264,17 @@ ENTRY(memcmp) ldreq lr, [r1], #4 ldreq r0, [r4], #4 orreq ip, ip, lr, lsl #16 - eoreqs r0, r0, ip + eorseq r0, r0, ip moveq ip, lr, lsr #16 ldreq lr, [r1], #4 ldreq r0, [r4], #4 orreq ip, ip, lr, lsl #16 - eoreqs r0, r0, ip + eorseq r0, r0, ip moveq ip, lr, lsr #16 ldreq lr, [r1], #4 ldreq r0, [r4], #4 orreq ip, ip, lr, lsl #16 - eoreqs r0, r0, ip + eorseq r0, r0, ip bne 7f subs r2, r2, #16 bhs 6b @@ -317,7 +318,7 @@ ENTRY(memcmp) ldreq r7, [r1], #4 ldreq r0, [r4], #4 orreq ip, ip, r7, lsl r6 - eoreqs r0, r0, ip + eorseq r0, r0, ip bne 7f subs r2, r2, #8 bhs 6b diff --git a/libc/arch-arm/bionic/memcpy.a9.S b/libc/arch-arm/bionic/memcpy.neon.S similarity index 98% rename from libc/arch-arm/bionic/memcpy.a9.S rename to libc/arch-arm/bionic/memcpy.neon.S index 259701d67..91c9cc6e0 100644 --- a/libc/arch-arm/bionic/memcpy.a9.S +++ b/libc/arch-arm/bionic/memcpy.neon.S @@ -27,7 +27,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ /* @@ -39,11 +38,9 @@ ARMv6 (ARMv7-a if using Neon) ARM state Unaligned accesses - LDRD/STRD support unaligned word accesses */ -#include #include .syntax unified @@ -250,9 +247,9 @@ ENTRY(memcpy) vmov.f32 s0, s0 #endif - /* SRC and DST have the same mutual 32-bit alignment, but we may + /* SRC and DST have the same mutual 64-bit alignment, but we may still need to pre-copy some bytes to get to natural alignment. - We bring DST into full 64-bit alignment. */ + We bring SRC and DST into full 64-bit alignment. */ lsls tmp2, dst, #29 beq 1f rsbs tmp2, tmp2, #0 @@ -351,7 +348,7 @@ ENTRY(memcpy) .Ltail63aligned: /* Count in tmp2. */ /* Copy up to 7 d-words of data. Similar to Ltail63unaligned, but - we know that the src and dest are 32-bit aligned so we can use + we know that the src and dest are 64-bit aligned so we can use LDRD/STRD to improve efficiency. */ /* TMP2 is now negative, but we don't care about that. The bottom six bits still tell us how many bytes are left to copy. */ @@ -611,4 +608,5 @@ ENTRY(memcpy) ldr tmp2, [sp], #FRAME_SIZE bne .Ltail63unaligned bx lr + END(memcpy) diff --git a/libc/arch-arm/bionic/strchr.S b/libc/arch-arm/bionic/strchr.S new file mode 100644 index 000000000..75abf51b9 --- /dev/null +++ b/libc/arch-arm/bionic/strchr.S @@ -0,0 +1,115 @@ +/*- + * Copyright (c) 2013 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Matt Thomas of 3am Software Foundry. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#ifndef __ARMEB__ +#define BYTE0 0x000000ff +#define BYTE1 0x0000ff00 +#define BYTE2 0x00ff0000 +#define BYTE3 0xff000000 +#define lshi lsl +#define lshis lsls +#else +#define BYTE0 0xff000000 +#define BYTE1 0x00ff0000 +#define BYTE2 0x0000ff00 +#define BYTE3 0x000000ff +#define lshi lsr +#define lshis lsrs +#endif + + .syntax unified + .arch armv7-a + + .thumb + .thumb_func + +ENTRY(strchr) + and r2, r1, #0xff /* restrict to byte value */ +1: tst r0, #3 /* test for word alignment */ + beq .Lpre_main_loop /* finally word aligned */ + ldrb r3, [r0], #1 /* load a byte */ + cmp r3, r2 /* is it a match? */ + beq 2f /* yes, return current ptr - 1 */ + cmp r3, #0 /* no, was it 0? */ + bne 1b /* no, try next byte */ + movs r0, #0 /* yes, set return value to NULL */ + bx lr /* return */ +2: subs r0, r0, #1 /* back up by one */ + bx lr /* return */ +.Lpre_main_loop: + movw ip, #0xfefe /* magic constant; 254 in each byte */ + movt ip, #0xfefe /* magic constant; 254 in each byte */ + orr r2, r2, r2, lsl #8 /* move to next byte */ + orr r2, r2, r2, lsl #16 /* move to next halfword */ +.Lmain_loop: + ldr r3, [r0], #4 /* load next word */ + /* + * Add 254 to each byte using the UQADD8 (unsigned saturating add 8) + * instruction. For every non-NUL byte, the result for that byte will + * become 255. For NUL, it will be 254. When we complement the + * result, if the result is non-0 then we must have encountered a NUL. + */ + uqadd8 r1, r3, ip /* NUL detection happens here */ + eors r3, r3, r2 /* xor to clear each lane */ + uqadd8 r3, r3, ip /* char detection happens here */ + ands r3, r3, r1 /* merge results */ + mvns r3, r3 /* is the complement non-0? */ + beq .Lmain_loop /* no, then keep going */ + + /* + * We've encountered a NUL or a match but we don't know which happened + * first. + */ + cmp r2, #0 /* searching for NUL? */ + beq .Lfind_match /* yes, find the match */ + mvns r1, r1 /* did we encounter a NUL? */ + beq .Lfind_match /* no, find the match */ + bics r3, r3, r1 /* clear match for the NUL(s) */ + beq .Lnomatch /* any left set? if not, no match */ + lshis r1, r1, #8 /* replicate NUL bit to other bytes */ + itt ne + orrne r1, r1, r1, lshi #8 /* replicate NUL bit to other bytes */ + orrne r1, r1, r1, lshi #8 /* replicate NUL bit to other bytes */ + bics r3, r3, r1 /* clear any match bits after the NUL */ + beq .Lnomatch /* any left set? if not, no match */ +.Lfind_match: +#ifndef __ARMEB__ + rev r3, r3 /* we want this in BE for the CLZ */ +#endif + clz r3, r3 /* count how many leading zeros */ + add r0, r0, r3, lsr #3 /* divide that by 8 and add to count */ + subs r0, r0, #4 /* compensate for the post-inc */ + bx lr +.Lnomatch: + movs r0, #0 + bx lr +END(strchr) diff --git a/libc/arch-arm/bionic/strrchr.S b/libc/arch-arm/bionic/strrchr.S new file mode 100644 index 000000000..56d37d45c --- /dev/null +++ b/libc/arch-arm/bionic/strrchr.S @@ -0,0 +1,131 @@ +/*- + * Copyright (c) 2013 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Matt Thomas of 3am Software Foundry. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#ifndef __ARMEB__ +#define BYTE0 0x000000ff +#define BYTE1 0x0000ff00 +#define BYTE2 0x00ff0000 +#define BYTE3 0xff000000 +#define lshi lsl +#define lshis lsls +#else +#define BYTE0 0xff000000 +#define BYTE1 0x00ff0000 +#define BYTE2 0x0000ff00 +#define BYTE3 0x000000ff +#define lshi lsr +#define lshis lsrs +#endif + + .syntax unified + .arch armv7-a + + .thumb + .thumb_func + +ENTRY(strrchr) + ands r2, r1, #0xff /* is the byte value NUL? */ + bne 1f /* no, do it the hard way */ + push {r0, lr} /* save pointer and return addr */ + bl strlen /* get length */ + pop {r1, r2} /* restore pointer / return addr */ + adds r0, r0, r1 /* add pointer to length */ + bx r2 /* return */ + +1: mov r1, r0 /* we use r0 at the return value */ + movs r0, #0 /* return NULL by default */ +2: tst r1, #3 /* test for word alignment */ + beq .Lpre_main_loop /* finally word aligned */ + ldrb r3, [r1], #1 /* load a byte */ + cmp r3, r2 /* did it match? */ + it eq + subeq r0, r1, #1 /* yes, remember that it did */ + cmp r3, #0 /* was it NUL? */ + bne 2b /* no, try next byte */ + bx lr /* return */ +.Lpre_main_loop: + push {r4, r5} /* save some registers */ + movw ip, #0xfefe /* magic constant; 254 in each byte */ + movt ip, #0xfefe /* magic constant; 254 in each byte */ + orr r2, r2, r2, lsl #8 /* move to next byte */ + orr r2, r2, r2, lsl #16 /* move to next halfword */ +.Lmain_loop: + ldr r3, [r1], #4 /* load next word */ + /* + * Add 254 to each byte using the UQADD8 (unsigned saturating add 8) + * instruction. For every non-NUL byte, the result for that byte will + * become 255. For NUL, it will be 254. When we complement the + * result, if the result is non-0 then we must have encountered a NUL. + */ + uqadd8 r4, r3, ip /* NUL detection happens here */ + usub8 r3, r3, r2 /* bias for char looked for? */ + uqadd8 r5, r3, ip /* char detection happens here */ + ands r3, r4, r5 /* merge results */ + mvns r3, r3 /* is the complement non-0? */ + beq .Lmain_loop /* no, then keep going */ + + mvns r5, r5 /* get we find any matching bytes? */ + beq .Ldone /* no, then we hit the end, return */ + mvns r4, r4 /* did we encounter a NUL? */ + beq .Lfind_match /* no, find matching byte */ + /* + * Copy the NUL bit to the following byte lanes. Then clear any match + * bits in those byte lanes to prevent false positives in those bytes. + */ + bics r5, r5, r4 /* clear any NUL match bits */ + beq .Ldone /* no remaining matches, we're done */ + lshis r3, r4, #8 /* shift up a byte */ + itt ne + orrsne r3, r3, r3, lshi #8 /* if non 0, copy up to next byte */ + orrsne r3, r3, r3, lshi #8 /* if non 0, copy up to last byte */ + bics r5, r5, r3 /* clear match bits */ + beq .Ldone /* no remaining matches, we're done */ +.Lfind_match: +#ifndef __ARMEB__ + rev r5, r5 /* we want this in BE for the CLZ */ +#endif + /* + * If we have multiple matches, we want to the select the "last" match + * in the word which will be the lowest bit set. + */ + subs r3, r5, #1 /* subtract 1 */ + ands r3, r3, r5 /* and with mask */ + eors r5, r5, r3 /* only have the lowest bit set left */ + clz r5, r5 /* count how many leading zeros */ + add r0, r1, r5, lsr #3 /* divide that by 8 and add to count */ + subs r0, r0, #4 /* compensate for the post-inc */ + cmp r4, #0 /* did we read any NULs? */ + beq .Lmain_loop /* no, get next word */ +.Ldone: + pop {r4, r5} + bx lr +END(strrchr) diff --git a/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S index a2e9c229e..3692f0425 100644 --- a/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S +++ b/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 The Android Open Source Project + * Copyright (C) 2015 The Android Open Source Project * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,191 +26,7 @@ * SUCH DAMAGE. */ -#include -#include +// Indicate which memcpy base file to include. +#define MEMCPY_BASE "memcpy_base.S" - .syntax unified - - .thumb - .thumb_func - -// Get the length of src string, then get the source of the dst string. -// Check that the two lengths together don't exceed the threshold, then -// do a memcpy of the data. -ENTRY(__strcat_chk) - pld [r0, #0] - push {r0, lr} - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - push {r4, r5} - .cfi_adjust_cfa_offset 8 - .cfi_rel_offset r4, 0 - .cfi_rel_offset r5, 4 - - mov lr, r2 - - // Save the dst register to r5 - mov r5, r0 - - // Zero out r4 - eor r4, r4, r4 - - // r1 contains the address of the string to count. -.L_strlen_start: - mov r0, r1 - ands r3, r1, #7 - beq .L_mainloop - - // Align to a double word (64 bits). - rsb r3, r3, #8 - lsls ip, r3, #31 - beq .L_align_to_32 - - ldrb r2, [r1], #1 - cbz r2, .L_update_count_and_finish - -.L_align_to_32: - bcc .L_align_to_64 - ands ip, r3, #2 - beq .L_align_to_64 - - ldrb r2, [r1], #1 - cbz r2, .L_update_count_and_finish - ldrb r2, [r1], #1 - cbz r2, .L_update_count_and_finish - -.L_align_to_64: - tst r3, #4 - beq .L_mainloop - ldr r3, [r1], #4 - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne .L_zero_in_second_register - - .p2align 2 -.L_mainloop: - ldrd r2, r3, [r1], #8 - - pld [r1, #64] - - sub ip, r2, #0x01010101 - bic ip, ip, r2 - ands ip, ip, #0x80808080 - bne .L_zero_in_first_register - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne .L_zero_in_second_register - b .L_mainloop - -.L_update_count_and_finish: - sub r3, r1, r0 - sub r3, r3, #1 - b .L_finish - -.L_zero_in_first_register: - sub r3, r1, r0 - lsls r2, ip, #17 - bne .L_sub8_and_finish - bcs .L_sub7_and_finish - lsls ip, ip, #1 - bne .L_sub6_and_finish - - sub r3, r3, #5 - b .L_finish - -.L_sub8_and_finish: - sub r3, r3, #8 - b .L_finish - -.L_sub7_and_finish: - sub r3, r3, #7 - b .L_finish - -.L_sub6_and_finish: - sub r3, r3, #6 - b .L_finish - -.L_zero_in_second_register: - sub r3, r1, r0 - lsls r2, ip, #17 - bne .L_sub4_and_finish - bcs .L_sub3_and_finish - lsls ip, ip, #1 - bne .L_sub2_and_finish - - sub r3, r3, #1 - b .L_finish - -.L_sub4_and_finish: - sub r3, r3, #4 - b .L_finish - -.L_sub3_and_finish: - sub r3, r3, #3 - b .L_finish - -.L_sub2_and_finish: - sub r3, r3, #2 - -.L_finish: - cmp r4, #0 - bne .L_strlen_done - - // Time to get the dst string length. - mov r1, r5 - - // Save the original source address to r5. - mov r5, r0 - - // Save the current length (adding 1 for the terminator). - add r4, r3, #1 - b .L_strlen_start - - // r0 holds the pointer to the dst string. - // r3 holds the dst string length. - // r4 holds the src string length + 1. -.L_strlen_done: - add r2, r3, r4 - cmp r2, lr - bhi __strcat_chk_failed - - // Set up the registers for the memcpy code. - mov r1, r5 - pld [r1, #64] - mov r2, r4 - add r0, r0, r3 - pop {r4, r5} -END(__strcat_chk) - -#define MEMCPY_BASE __strcat_chk_memcpy_base -#define MEMCPY_BASE_ALIGNED __strcat_chk_memcpy_base_aligned - -#include "memcpy_base.S" - -ENTRY_PRIVATE(__strcat_chk_failed) - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - .cfi_adjust_cfa_offset 8 - .cfi_rel_offset r4, 0 - .cfi_rel_offset r5, 4 - - ldr r0, error_message - ldr r1, error_code -1: - add r0, pc - bl __fortify_chk_fail -error_code: - .word BIONIC_EVENT_STRCAT_BUFFER_OVERFLOW -error_message: - .word error_string-(1b+4) -END(__strcat_chk_failed) - - .data -error_string: - .string "strcat: prevented write past end of buffer" +#include "__strcat_chk_common.S" diff --git a/libc/arch-arm/cortex-a15/bionic/__strcat_chk_common.S b/libc/arch-arm/cortex-a15/bionic/__strcat_chk_common.S new file mode 100644 index 000000000..de66967ee --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/__strcat_chk_common.S @@ -0,0 +1,212 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + + .syntax unified + + .thumb + .thumb_func + +// Get the length of src string, then get the source of the dst string. +// Check that the two lengths together don't exceed the threshold, then +// do a memcpy of the data. +ENTRY(__strcat_chk) + pld [r0, #0] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + push {r4, r5} + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 + + mov lr, r2 + + // Save the dst register to r5 + mov r5, r0 + + // Zero out r4 + eor r4, r4, r4 + + // r1 contains the address of the string to count. +.L_strlen_start: + mov r0, r1 + ands r3, r1, #7 + beq .L_mainloop + + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq .L_align_to_32 + + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_32: + bcc .L_align_to_64 + ands ip, r3, #2 + beq .L_align_to_64 + + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_64: + tst r3, #4 + beq .L_mainloop + ldr r3, [r1], #4 + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + + .p2align 2 +.L_mainloop: + ldrd r2, r3, [r1], #8 + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne .L_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + b .L_mainloop + +.L_update_count_and_finish: + sub r3, r1, r0 + sub r3, r3, #1 + b .L_finish + +.L_zero_in_first_register: + sub r3, r1, r0 + lsls r2, ip, #17 + bne .L_sub8_and_finish + bcs .L_sub7_and_finish + lsls ip, ip, #1 + bne .L_sub6_and_finish + + sub r3, r3, #5 + b .L_finish + +.L_sub8_and_finish: + sub r3, r3, #8 + b .L_finish + +.L_sub7_and_finish: + sub r3, r3, #7 + b .L_finish + +.L_sub6_and_finish: + sub r3, r3, #6 + b .L_finish + +.L_zero_in_second_register: + sub r3, r1, r0 + lsls r2, ip, #17 + bne .L_sub4_and_finish + bcs .L_sub3_and_finish + lsls ip, ip, #1 + bne .L_sub2_and_finish + + sub r3, r3, #1 + b .L_finish + +.L_sub4_and_finish: + sub r3, r3, #4 + b .L_finish + +.L_sub3_and_finish: + sub r3, r3, #3 + b .L_finish + +.L_sub2_and_finish: + sub r3, r3, #2 + +.L_finish: + cmp r4, #0 + bne .L_strlen_done + + // Time to get the dst string length. + mov r1, r5 + + // Save the original source address to r5. + mov r5, r0 + + // Save the current length (adding 1 for the terminator). + add r4, r3, #1 + b .L_strlen_start + + // r0 holds the pointer to the dst string. + // r3 holds the dst string length. + // r4 holds the src string length + 1. +.L_strlen_done: + add r2, r3, r4 + cmp r2, lr + bhi .L_strcat_chk_failed + + // Set up the registers for the memcpy code. + mov r1, r5 + pld [r1, #64] + mov r2, r4 + add r0, r0, r3 + pop {r4, r5} + .cfi_adjust_cfa_offset -8 + .cfi_restore r4 + .cfi_restore r5 + +#include MEMCPY_BASE + + // Undo the above cfi directives + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 +.L_strcat_chk_failed: + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_STRCAT_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+4) +END(__strcat_chk) + + .data +error_string: + .string "strcat: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S index db766863a..d8cb3d9d7 100644 --- a/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S +++ b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 The Android Open Source Project + * Copyright (C) 2015 The Android Open Source Project * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,155 +26,7 @@ * SUCH DAMAGE. */ -#include -#include +// Indicate which memcpy base file to include. +#define MEMCPY_BASE "memcpy_base.S" - .syntax unified - - .thumb - .thumb_func - -// Get the length of the source string first, then do a memcpy of the data -// instead of a strcpy. -ENTRY(__strcpy_chk) - pld [r0, #0] - push {r0, lr} - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - - mov lr, r2 - mov r0, r1 - - ands r3, r1, #7 - beq .L_mainloop - - // Align to a double word (64 bits). - rsb r3, r3, #8 - lsls ip, r3, #31 - beq .L_align_to_32 - - ldrb r2, [r0], #1 - cbz r2, .L_update_count_and_finish - -.L_align_to_32: - bcc .L_align_to_64 - ands ip, r3, #2 - beq .L_align_to_64 - - ldrb r2, [r0], #1 - cbz r2, .L_update_count_and_finish - ldrb r2, [r0], #1 - cbz r2, .L_update_count_and_finish - -.L_align_to_64: - tst r3, #4 - beq .L_mainloop - ldr r3, [r0], #4 - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne .L_zero_in_second_register - - .p2align 2 -.L_mainloop: - ldrd r2, r3, [r0], #8 - - pld [r0, #64] - - sub ip, r2, #0x01010101 - bic ip, ip, r2 - ands ip, ip, #0x80808080 - bne .L_zero_in_first_register - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne .L_zero_in_second_register - b .L_mainloop - -.L_update_count_and_finish: - sub r3, r0, r1 - sub r3, r3, #1 - b .L_check_size - -.L_zero_in_first_register: - sub r3, r0, r1 - lsls r2, ip, #17 - bne .L_sub8_and_finish - bcs .L_sub7_and_finish - lsls ip, ip, #1 - bne .L_sub6_and_finish - - sub r3, r3, #5 - b .L_check_size - -.L_sub8_and_finish: - sub r3, r3, #8 - b .L_check_size - -.L_sub7_and_finish: - sub r3, r3, #7 - b .L_check_size - -.L_sub6_and_finish: - sub r3, r3, #6 - b .L_check_size - -.L_zero_in_second_register: - sub r3, r0, r1 - lsls r2, ip, #17 - bne .L_sub4_and_finish - bcs .L_sub3_and_finish - lsls ip, ip, #1 - bne .L_sub2_and_finish - - sub r3, r3, #1 - b .L_check_size - -.L_sub4_and_finish: - sub r3, r3, #4 - b .L_check_size - -.L_sub3_and_finish: - sub r3, r3, #3 - b .L_check_size - -.L_sub2_and_finish: - sub r3, r3, #2 - -.L_check_size: - pld [r1, #0] - pld [r1, #64] - ldr r0, [sp] - cmp r3, lr - bhs __strcpy_chk_failed - - // Add 1 for copy length to get the string terminator. - add r2, r3, #1 -END(__strcpy_chk) - -#define MEMCPY_BASE __strcpy_chk_memcpy_base -#define MEMCPY_BASE_ALIGNED __strcpy_chk_memcpy_base_aligned -#include "memcpy_base.S" - -ENTRY_PRIVATE(__strcpy_chk_failed) - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - - ldr r0, error_message - ldr r1, error_code -1: - add r0, pc - bl __fortify_chk_fail -error_code: - .word BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW -error_message: - .word error_string-(1b+4) -END(__strcpy_chk_failed) - - .data -error_string: - .string "strcpy: prevented write past end of buffer" +#include "__strcpy_chk_common.S" diff --git a/libc/arch-arm/cortex-a15/bionic/__strcpy_chk_common.S b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk_common.S new file mode 100644 index 000000000..69ebcb497 --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk_common.S @@ -0,0 +1,173 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + + .syntax unified + + .thumb + .thumb_func + +// Get the length of the source string first, then do a memcpy of the data +// instead of a strcpy. +ENTRY(__strcpy_chk) + pld [r0, #0] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + + mov lr, r2 + mov r0, r1 + + ands r3, r1, #7 + beq .L_mainloop + + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq .L_align_to_32 + + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_32: + bcc .L_align_to_64 + ands ip, r3, #2 + beq .L_align_to_64 + + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_64: + tst r3, #4 + beq .L_mainloop + ldr r3, [r0], #4 + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + + .p2align 2 +.L_mainloop: + ldrd r2, r3, [r0], #8 + + pld [r0, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne .L_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + b .L_mainloop + +.L_update_count_and_finish: + sub r3, r0, r1 + sub r3, r3, #1 + b .L_check_size + +.L_zero_in_first_register: + sub r3, r0, r1 + lsls r2, ip, #17 + bne .L_sub8_and_finish + bcs .L_sub7_and_finish + lsls ip, ip, #1 + bne .L_sub6_and_finish + + sub r3, r3, #5 + b .L_check_size + +.L_sub8_and_finish: + sub r3, r3, #8 + b .L_check_size + +.L_sub7_and_finish: + sub r3, r3, #7 + b .L_check_size + +.L_sub6_and_finish: + sub r3, r3, #6 + b .L_check_size + +.L_zero_in_second_register: + sub r3, r0, r1 + lsls r2, ip, #17 + bne .L_sub4_and_finish + bcs .L_sub3_and_finish + lsls ip, ip, #1 + bne .L_sub2_and_finish + + sub r3, r3, #1 + b .L_check_size + +.L_sub4_and_finish: + sub r3, r3, #4 + b .L_check_size + +.L_sub3_and_finish: + sub r3, r3, #3 + b .L_check_size + +.L_sub2_and_finish: + sub r3, r3, #2 + +.L_check_size: + pld [r1, #0] + pld [r1, #64] + ldr r0, [sp] + cmp r3, lr + bhs .L_strcpy_chk_failed + + // Add 1 for copy length to get the string terminator. + add r2, r3, #1 + +#include MEMCPY_BASE + +.L_strcpy_chk_failed: + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+4) +END(__strcpy_chk) + + .data +error_string: + .string "strcpy: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a15/bionic/memchr.S b/libc/arch-arm/cortex-a15/bionic/memchr.S new file mode 100644 index 000000000..328fe81e8 --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/memchr.S @@ -0,0 +1,152 @@ +/* Copyright (c) 2010-2011, Linaro Limited + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Linaro Limited nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + Written by Dave Gilbert + Adapted to work inside Bionic by Bernhard Rosenkränzer + + This memchr routine is optimised on a Cortex-A9 and should work on + all ARMv7 processors. It has a fast past for short sizes, and has + an optimised path for large data sets; the worst case is finding the + match early in a large data set. + + */ + +// Prototype: void *memchr(const void *s, int c, size_t n); + +#include +#include "private/libc_events.h" + + .syntax unified + .arch armv7-a + +@ this lets us check a flag in a 00/ff byte easily in either endianness +#ifdef __ARMEB__ +#define CHARTSTMASK(c) 1<<(31-(c*8)) +#else +#define CHARTSTMASK(c) 1<<(c*8) +#endif + .thumb + +@ --------------------------------------------------------------------------- +ENTRY(memchr) + .thumb_func + .align 2 + .p2align 4,,15 + @ r0 = start of memory to scan + @ r1 = character to look for + @ r2 = length + @ returns r0 = pointer to character or NULL if not found + and r1,r1,#0xff @ Don't think we can trust the caller to actually pass a char + + cmp r2,#16 @ If it's short don't bother with anything clever + blt 20f + + tst r0, #7 @ If it's already aligned skip the next bit + beq 10f + + @ Work up to an aligned point +5: + ldrb r3, [r0],#1 + subs r2, r2, #1 + cmp r3, r1 + beq 50f @ If it matches exit found + tst r0, #7 + bne 5b @ If not aligned yet then do next byte + +10: + @ At this point, we are aligned, we know we have at least 8 bytes to work with + push {r4,r5,r6,r7} + orr r1, r1, r1, lsl #8 @ expand the match word across to all bytes + orr r1, r1, r1, lsl #16 + bic r4, r2, #7 @ Number of double words to work with + mvns r7, #0 @ all F's + movs r3, #0 + +15: + ldrd r5,r6,[r0],#8 + subs r4, r4, #8 + eor r5,r5, r1 @ Get it so that r5,r6 have 00's where the bytes match the target + eor r6,r6, r1 + uadd8 r5, r5, r7 @ Parallel add 0xff - sets the GE bits for anything that wasn't 0 + sel r5, r3, r7 @ bytes are 00 for none-00 bytes, or ff for 00 bytes - NOTE INVERSION + uadd8 r6, r6, r7 @ Parallel add 0xff - sets the GE bits for anything that wasn't 0 + sel r6, r5, r7 @ chained....bytes are 00 for none-00 bytes, or ff for 00 bytes - NOTE INVERSION + cbnz r6, 60f + bne 15b @ (Flags from the subs above) If not run out of bytes then go around again + + pop {r4,r5,r6,r7} + and r1,r1,#0xff @ Get r1 back to a single character from the expansion above + and r2,r2,#7 @ Leave the count remaining as the number after the double words have been done + +20: + cbz r2, 40f @ 0 length or hit the end already then not found + +21: @ Post aligned section, or just a short call + ldrb r3,[r0],#1 + subs r2,r2,#1 + eor r3,r3,r1 @ r3 = 0 if match - doesn't break flags from sub + cbz r3, 50f + bne 21b @ on r2 flags + +40: + movs r0,#0 @ not found + bx lr + +50: + subs r0,r0,#1 @ found + bx lr + +60: @ We're here because the fast path found a hit - now we have to track down exactly which word it was + @ r0 points to the start of the double word after the one that was tested + @ r5 has the 00/ff pattern for the first word, r6 has the chained value + cmp r5, #0 + itte eq + moveq r5, r6 @ the end is in the 2nd word + subeq r0,r0,#3 @ Points to 2nd byte of 2nd word + subne r0,r0,#7 @ or 2nd byte of 1st word + + @ r0 currently points to the 3rd byte of the word containing the hit + tst r5, # CHARTSTMASK(0) @ 1st character + bne 61f + adds r0,r0,#1 + tst r5, # CHARTSTMASK(1) @ 2nd character + ittt eq + addeq r0,r0,#1 + tsteq r5, # (3<<15) @ 2nd & 3rd character + @ If not the 3rd must be the last one + addeq r0,r0,#1 + +61: + pop {r4,r5,r6,r7} + subs r0,r0,#1 + bx lr +END(memchr) diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy.S b/libc/arch-arm/cortex-a15/bionic/memcpy.S index 410b663e2..537f3de2f 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy.S @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008 The Android Open Source Project + * Copyright (C) 2015 The Android Open Source Project * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -25,79 +25,8 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ -/* - * Copyright (c) 2013 ARM Ltd - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the company may not be used to endorse or promote - * products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -// Prototype: void *memcpy (void *dst, const void *src, size_t count). - -#include -#include - - .text - .syntax unified - .fpu neon - -ENTRY(__memcpy_chk) - cmp r2, r3 - bhi __memcpy_chk_fail - - // Fall through to memcpy... -END(__memcpy_chk) - -ENTRY(memcpy) - pld [r1, #64] - push {r0, lr} - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 -END(memcpy) - -#define MEMCPY_BASE __memcpy_base -#define MEMCPY_BASE_ALIGNED __memcpy_base_aligned -#include "memcpy_base.S" - -ENTRY_PRIVATE(__memcpy_chk_fail) - // Preserve lr for backtrace. - push {lr} - .cfi_def_cfa_offset 4 - .cfi_rel_offset lr, 0 - ldr r0, error_message - ldr r1, error_code -1: - add r0, pc - bl __fortify_chk_fail -error_code: - .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW -error_message: - .word error_string-(1b+8) -END(__memcpy_chk_fail) +// Indicate which memcpy base file to include. +#define MEMCPY_BASE "memcpy_base.S" - .data -error_string: - .string "memcpy: prevented write past end of buffer" +#include "memcpy_common.S" diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy_base.S b/libc/arch-arm/cortex-a15/bionic/memcpy_base.S index 2a7385247..aac737df5 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy_base.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy_base.S @@ -53,11 +53,7 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -ENTRY_PRIVATE(MEMCPY_BASE) - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - +.L_memcpy_base: // Assumes that n >= 0, and dst, src are valid pointers. // For any sizes less than 832 use the neon code that doesn't // care about the src alignment. This avoids any checks @@ -168,12 +164,6 @@ ENTRY_PRIVATE(MEMCPY_BASE) eor r3, r0, r1 ands r3, r3, #0x3 bne .L_copy_unknown_alignment -END(MEMCPY_BASE) - -ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED) - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 // To try and improve performance, stack layout changed, // i.e., not keeping the stack looking like users expect @@ -185,7 +175,7 @@ ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED) strd r6, r7, [sp, #-8]! .cfi_adjust_cfa_offset 8 .cfi_rel_offset r6, 0 - .cfi_rel_offset r7, 0 + .cfi_rel_offset r7, 4 strd r8, r9, [sp, #-8]! .cfi_adjust_cfa_offset 8 .cfi_rel_offset r8, 0 @@ -291,10 +281,28 @@ ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED) // Restore registers: optimized pop {r0, pc} ldrd r8, r9, [sp], #8 + .cfi_adjust_cfa_offset -8 + .cfi_restore r8 + .cfi_restore r9 ldrd r6, r7, [sp], #8 + .cfi_adjust_cfa_offset -8 + .cfi_restore r6 + .cfi_restore r7 ldrd r4, r5, [sp], #8 + .cfi_adjust_cfa_offset -8 + .cfi_restore r4 + .cfi_restore r5 pop {r0, pc} + // Put the cfi directives back for the below instructions. + .cfi_adjust_cfa_offset 24 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 + .cfi_rel_offset r6, 8 + .cfi_rel_offset r7, 12 + .cfi_rel_offset r8, 16 + .cfi_rel_offset r9, 20 + .L_dst_not_word_aligned: // Align dst to word. rsb ip, ip, #4 @@ -315,4 +323,12 @@ ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED) // Src is guaranteed to be at least word aligned by this point. b .L_word_aligned -END(MEMCPY_BASE_ALIGNED) + + // Undo any cfi directives from above. + .cfi_adjust_cfa_offset -24 + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 + .cfi_restore r8 + .cfi_restore r9 diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy_common.S b/libc/arch-arm/cortex-a15/bionic/memcpy_common.S new file mode 100644 index 000000000..464fb46b3 --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/memcpy_common.S @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + + .text + .syntax unified + .fpu neon + +ENTRY(__memcpy_chk) + cmp r2, r3 + bhi .L_memcpy_chk_fail + + // Fall through to memcpy... +END(__memcpy_chk) + +// Prototype: void *memcpy (void *dst, const void *src, size_t count). +ENTRY(memcpy) + pld [r1, #64] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + +#include MEMCPY_BASE + + // Undo the cfi instructions from above. + .cfi_def_cfa_offset 0 + .cfi_restore r0 + .cfi_restore lr +.L_memcpy_chk_fail: + // Preserve lr for backtrace. + push {lr} + .cfi_adjust_cfa_offset 4 + .cfi_rel_offset lr, 0 + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) +END(memcpy) + + .data +error_string: + .string "memcpy: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a15/bionic/strlen.S b/libc/arch-arm/cortex-a15/bionic/strlen.S index 9a0ce62a9..8545132fb 100644 --- a/libc/arch-arm/cortex-a15/bionic/strlen.S +++ b/libc/arch-arm/cortex-a15/bionic/strlen.S @@ -1,165 +1,146 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. +/* Copyright (c) 2010-2011,2013 Linaro Limited + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Linaro Limited nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + /* - * Copyright (c) 2013 ARM Ltd - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the company may not be used to endorse or promote - * products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + Assumes: + ARMv6T2, AArch32 + + Adapted to Bionic by Bernhard Rosenkränzer */ #include - .syntax unified - - .thumb - .thumb_func +#ifdef __ARMEB__ +#define S2LO lsl +#define S2HI lsr +#else +#define S2LO lsr +#define S2HI lsl +#endif + + .text + /* This code requires Thumb. */ + .thumb + .syntax unified + +/* Parameters and result. */ +#define srcin r0 +#define result r0 + +/* Internal variables. */ +#define src r1 +#define data1a r2 +#define data1b r3 +#define const_m1 r12 +#define const_0 r4 +#define tmp1 r4 /* Overlaps const_0 */ +#define tmp2 r5 ENTRY(strlen) - pld [r0, #0] - mov r1, r0 - - ands r3, r0, #7 - beq mainloop - - // Align to a double word (64 bits). - rsb r3, r3, #8 - lsls ip, r3, #31 - beq align_to_32 - - ldrb r2, [r1], #1 - cbz r2, update_count_and_return - -align_to_32: - bcc align_to_64 - ands ip, r3, #2 - beq align_to_64 - - ldrb r2, [r1], #1 - cbz r2, update_count_and_return - ldrb r2, [r1], #1 - cbz r2, update_count_and_return - -align_to_64: - tst r3, #4 - beq mainloop - ldr r3, [r1], #4 - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne zero_in_second_register - - .p2align 2 -mainloop: - ldrd r2, r3, [r1], #8 - - pld [r1, #64] - - sub ip, r2, #0x01010101 - bic ip, ip, r2 - ands ip, ip, #0x80808080 - bne zero_in_first_register - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne zero_in_second_register - b mainloop - -update_count_and_return: - sub r0, r1, r0 - sub r0, r0, #1 - bx lr - -zero_in_first_register: - sub r0, r1, r0 - lsls r3, ip, #17 - bne sub8_and_return - bcs sub7_and_return - lsls ip, ip, #1 - bne sub6_and_return - - sub r0, r0, #5 - bx lr - -sub8_and_return: - sub r0, r0, #8 - bx lr - -sub7_and_return: - sub r0, r0, #7 - bx lr - -sub6_and_return: - sub r0, r0, #6 - bx lr - -zero_in_second_register: - sub r0, r1, r0 - lsls r3, ip, #17 - bne sub4_and_return - bcs sub3_and_return - lsls ip, ip, #1 - bne sub2_and_return - - sub r0, r0, #1 - bx lr - -sub4_and_return: - sub r0, r0, #4 - bx lr - -sub3_and_return: - sub r0, r0, #3 - bx lr - -sub2_and_return: - sub r0, r0, #2 - bx lr + .p2align 6 + pld [srcin, #0] + strd r4, r5, [sp, #-8]! + bic src, srcin, #7 + mvn const_m1, #0 + ands tmp1, srcin, #7 /* (8 - bytes) to alignment. */ + pld [src, #32] + bne.w misaligned8 + mov const_0, #0 + mov result, #-8 +loop_aligned: + /* Bytes 0-7. */ + ldrd data1a, data1b, [src] + pld [src, #64] + add result, result, #8 +start_realigned: + uadd8 data1a, data1a, const_m1 /* Saturating GE<0:3> set. */ + sel data1a, const_0, const_m1 /* Select based on GE<0:3>. */ + uadd8 data1b, data1b, const_m1 + sel data1b, data1a, const_m1 /* Only used if d1a == 0. */ + cbnz data1b, null_found + + /* Bytes 8-15. */ + ldrd data1a, data1b, [src, #8] + uadd8 data1a, data1a, const_m1 /* Saturating GE<0:3> set. */ + add result, result, #8 + sel data1a, const_0, const_m1 /* Select based on GE<0:3>. */ + uadd8 data1b, data1b, const_m1 + sel data1b, data1a, const_m1 /* Only used if d1a == 0. */ + cbnz data1b, null_found + + /* Bytes 16-23. */ + ldrd data1a, data1b, [src, #16] + uadd8 data1a, data1a, const_m1 /* Saturating GE<0:3> set. */ + add result, result, #8 + sel data1a, const_0, const_m1 /* Select based on GE<0:3>. */ + uadd8 data1b, data1b, const_m1 + sel data1b, data1a, const_m1 /* Only used if d1a == 0. */ + cbnz data1b, null_found + + /* Bytes 24-31. */ + ldrd data1a, data1b, [src, #24] + add src, src, #32 + uadd8 data1a, data1a, const_m1 /* Saturating GE<0:3> set. */ + add result, result, #8 + sel data1a, const_0, const_m1 /* Select based on GE<0:3>. */ + uadd8 data1b, data1b, const_m1 + sel data1b, data1a, const_m1 /* Only used if d1a == 0. */ + cmp data1b, #0 + beq loop_aligned + +null_found: + cmp data1a, #0 + itt eq + addeq result, result, #4 + moveq data1a, data1b +#ifndef __ARMEB__ + rev data1a, data1a +#endif + clz data1a, data1a + ldrd r4, r5, [sp], #8 + add result, result, data1a, lsr #3 /* Bits -> Bytes. */ + bx lr + +misaligned8: + ldrd data1a, data1b, [src] + and tmp2, tmp1, #3 + rsb result, tmp1, #0 + lsl tmp2, tmp2, #3 /* Bytes -> bits. */ + tst tmp1, #4 + pld [src, #64] + S2HI tmp2, const_m1, tmp2 + orn data1a, data1a, tmp2 + itt ne + ornne data1b, data1b, tmp2 + movne data1a, const_m1 + mov const_0, #0 + b start_realigned END(strlen) diff --git a/libc/arch-arm/cortex-a15/cortex-a15.mk b/libc/arch-arm/cortex-a15/cortex-a15.mk index f1abe3231..d82c49cb3 100644 --- a/libc/arch-arm/cortex-a15/cortex-a15.mk +++ b/libc/arch-arm/cortex-a15/cortex-a15.mk @@ -1,7 +1,8 @@ libc_bionic_src_files_arm += \ - arch-arm/cortex-a15/bionic/memcpy.S \ + arch-arm/cortex-a15/bionic/memchr.S \ + bionic/__memcpy_chk.cpp \ + arch-arm/bionic/memcpy.neon.S \ arch-arm/cortex-a15/bionic/memset.S \ - arch-arm/cortex-a15/bionic/stpcpy.S \ arch-arm/cortex-a15/bionic/strcat.S \ arch-arm/cortex-a15/bionic/__strcat_chk.S \ arch-arm/cortex-a15/bionic/strcmp.S \ @@ -9,3 +10,12 @@ libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/__strcpy_chk.S \ arch-arm/cortex-a15/bionic/strlen.S \ bionic/memmove.c \ + +# Optimization not required for some targets +ifeq ($(TARGET_CPU_MEMCPY_OPT_DISABLE),true) +libc_bionic_src_files_arm += \ + arch-arm/cortex-a7/bionic/memcpy.S +else +libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memcpy.S +endif diff --git a/libc/arch-arm/cortex-a53/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a53/bionic/__strcat_chk.S new file mode 100644 index 000000000..c5bc98a72 --- /dev/null +++ b/libc/arch-arm/cortex-a53/bionic/__strcat_chk.S @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +// Indicate which memcpy base file to include. +#define MEMCPY_BASE "arch-arm/cortex-a53/bionic/memcpy_base.S" + +#include "arch-arm/cortex-a15/bionic/__strcat_chk_common.S" diff --git a/libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S new file mode 100644 index 000000000..1f8945d9f --- /dev/null +++ b/libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +// Indicate which memcpy base file to include. +#define MEMCPY_BASE "arch-arm/cortex-a53/bionic/memcpy_base.S" + +#include "arch-arm/cortex-a15/bionic/__strcpy_chk_common.S" diff --git a/libc/arch-arm/cortex-a53/bionic/memcpy.S b/libc/arch-arm/cortex-a53/bionic/memcpy.S new file mode 100644 index 000000000..664f57442 --- /dev/null +++ b/libc/arch-arm/cortex-a53/bionic/memcpy.S @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +// Indicate which memcpy base file to include. +#define MEMCPY_BASE "arch-arm/cortex-a53/bionic/memcpy_base.S" + +#include "arch-arm/cortex-a15/bionic/memcpy_common.S" diff --git a/libc/arch-arm/cortex-a53/bionic/memcpy_base.S b/libc/arch-arm/cortex-a53/bionic/memcpy_base.S new file mode 100644 index 000000000..2749fc83d --- /dev/null +++ b/libc/arch-arm/cortex-a53/bionic/memcpy_base.S @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +.L_memcpy_base: + // Assumes that n >= 0, and dst, src are valid pointers. + cmp r2, #16 + blo .L_copy_less_than_16_unknown_align + +.L_copy_unknown_alignment: + // Unknown alignment of src and dst. + // Assumes that the first few bytes have already been prefetched. + + // Align destination to 128 bits. The mainloop store instructions + // require this alignment or they will throw an exception. + rsb r3, r0, #0 + ands r3, r3, #0xF + beq 2f + + // Copy up to 15 bytes (count in r3). + sub r2, r2, r3 + movs ip, r3, lsl #31 + + itt mi + ldrbmi lr, [r1], #1 + strbmi lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1], #1 + strbcs ip, [r0], #1 + strbcs lr, [r0], #1 + + movs ip, r3, lsl #29 + bge 1f + // Copies 4 bytes, dst 32 bits aligned before, at least 64 bits after. + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! +1: bcc 2f + // Copies 8 bytes, dst 64 bits aligned before, at least 128 bits after. + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0, :64]! + +2: // Make sure we have at least 64 bytes to copy. + subs r2, r2, #64 + blo 2f + +1: // The main loop copies 64 bytes at a time. + vld1.8 {d0 - d3}, [r1]! + vld1.8 {d4 - d7}, [r1]! + subs r2, r2, #64 + vstmia r0!, {d0 - d7} + pld [r1, #(64*10)] + bhs 1b + +2: // Fix-up the remaining count and make sure we have >= 32 bytes left. + adds r2, r2, #32 + blo 3f + + // 32 bytes. These cache lines were already preloaded. + vld1.8 {d0 - d3}, [r1]! + sub r2, r2, #32 + vst1.8 {d0 - d3}, [r0, :128]! +3: // Less than 32 left. + add r2, r2, #32 + tst r2, #0x10 + beq .L_copy_less_than_16_unknown_align + // Copies 16 bytes, destination 128 bits aligned. + vld1.8 {d0, d1}, [r1]! + vst1.8 {d0, d1}, [r0, :128]! + +.L_copy_less_than_16_unknown_align: + // Copy up to 15 bytes (count in r2). + movs ip, r2, lsl #29 + bcc 1f + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0]! +1: bge 2f + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! + +2: // Copy 0 to 4 bytes. + lsls r2, r2, #31 + itt ne + ldrbne lr, [r1], #1 + strbne lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1] + strbcs ip, [r0], #1 + strbcs lr, [r0] + + pop {r0, pc} diff --git a/libc/arch-arm/cortex-a53/cortex-a53.mk b/libc/arch-arm/cortex-a53/cortex-a53.mk new file mode 100644 index 000000000..3f75ebbfc --- /dev/null +++ b/libc/arch-arm/cortex-a53/cortex-a53.mk @@ -0,0 +1,18 @@ +libc_bionic_src_files_arm += \ + arch-arm/cortex-a53/bionic/memcpy.S \ + arch-arm/cortex-a53/bionic/__strcat_chk.S \ + arch-arm/cortex-a53/bionic/__strcpy_chk.S \ + +libc_bionic_src_files_arm += \ + arch-arm/cortex-a7/bionic/memset.S \ + +libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/strcat.S \ + arch-arm/cortex-a15/bionic/strcmp.S \ + arch-arm/cortex-a15/bionic/strcpy.S \ + arch-arm/cortex-a15/bionic/strlen.S \ + arch-arm/cortex-a15/bionic/stpcpy.S \ + arch-arm/krait/bionic/memmove.S + +libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memchr.S diff --git a/libc/arch-arm/cortex-a7/bionic/memcpy.S b/libc/arch-arm/cortex-a7/bionic/memcpy.S new file mode 100644 index 000000000..410b663e2 --- /dev/null +++ b/libc/arch-arm/cortex-a7/bionic/memcpy.S @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// Prototype: void *memcpy (void *dst, const void *src, size_t count). + +#include +#include + + .text + .syntax unified + .fpu neon + +ENTRY(__memcpy_chk) + cmp r2, r3 + bhi __memcpy_chk_fail + + // Fall through to memcpy... +END(__memcpy_chk) + +ENTRY(memcpy) + pld [r1, #64] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 +END(memcpy) + +#define MEMCPY_BASE __memcpy_base +#define MEMCPY_BASE_ALIGNED __memcpy_base_aligned +#include "memcpy_base.S" + +ENTRY_PRIVATE(__memcpy_chk_fail) + // Preserve lr for backtrace. + push {lr} + .cfi_def_cfa_offset 4 + .cfi_rel_offset lr, 0 + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) +END(__memcpy_chk_fail) + + .data +error_string: + .string "memcpy: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a7/bionic/memcpy_base.S b/libc/arch-arm/cortex-a7/bionic/memcpy_base.S new file mode 100644 index 000000000..2a7385247 --- /dev/null +++ b/libc/arch-arm/cortex-a7/bionic/memcpy_base.S @@ -0,0 +1,318 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +ENTRY_PRIVATE(MEMCPY_BASE) + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + + // Assumes that n >= 0, and dst, src are valid pointers. + // For any sizes less than 832 use the neon code that doesn't + // care about the src alignment. This avoids any checks + // for src alignment, and offers the best improvement since + // smaller sized copies are dominated by the overhead of + // the pre and post main loop. + // For larger copies, if src and dst cannot both be aligned to + // word boundaries, use the neon code. + // For all other copies, align dst to a double word boundary + // and copy using LDRD/STRD instructions. + + cmp r2, #16 + blo .L_copy_less_than_16_unknown_align + + // TODO: The aligned copy code is extremely slow copying some large + // buffers so always go through the unaligned path for now. + //cmp r2, #832 + //bge .L_check_alignment + +.L_copy_unknown_alignment: + // Unknown alignment of src and dst. + // Assumes that the first few bytes have already been prefetched. + + // Align destination to 128 bits. The mainloop store instructions + // require this alignment or they will throw an exception. + rsb r3, r0, #0 + ands r3, r3, #0xF + beq 2f + + // Copy up to 15 bytes (count in r3). + sub r2, r2, r3 + movs ip, r3, lsl #31 + + itt mi + ldrbmi lr, [r1], #1 + strbmi lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1], #1 + strbcs ip, [r0], #1 + strbcs lr, [r0], #1 + + movs ip, r3, lsl #29 + bge 1f + // Copies 4 bytes, dst 32 bits aligned before, at least 64 bits after. + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! +1: bcc 2f + // Copies 8 bytes, dst 64 bits aligned before, at least 128 bits after. + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0, :64]! + +2: // Make sure we have at least 64 bytes to copy. + subs r2, r2, #64 + blo 2f + +1: // The main loop copies 64 bytes at a time. + vld1.8 {d0 - d3}, [r1]! + vld1.8 {d4 - d7}, [r1]! + pld [r1, #(64*4)] + subs r2, r2, #64 + vst1.8 {d0 - d3}, [r0, :128]! + vst1.8 {d4 - d7}, [r0, :128]! + bhs 1b + +2: // Fix-up the remaining count and make sure we have >= 32 bytes left. + adds r2, r2, #32 + blo 3f + + // 32 bytes. These cache lines were already preloaded. + vld1.8 {d0 - d3}, [r1]! + sub r2, r2, #32 + vst1.8 {d0 - d3}, [r0, :128]! +3: // Less than 32 left. + add r2, r2, #32 + tst r2, #0x10 + beq .L_copy_less_than_16_unknown_align + // Copies 16 bytes, destination 128 bits aligned. + vld1.8 {d0, d1}, [r1]! + vst1.8 {d0, d1}, [r0, :128]! + +.L_copy_less_than_16_unknown_align: + // Copy up to 15 bytes (count in r2). + movs ip, r2, lsl #29 + bcc 1f + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0]! +1: bge 2f + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! + +2: // Copy 0 to 4 bytes. + lsls r2, r2, #31 + itt ne + ldrbne lr, [r1], #1 + strbne lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1] + strbcs ip, [r0], #1 + strbcs lr, [r0] + + pop {r0, pc} + +.L_check_alignment: + // If src and dst cannot both be aligned to a word boundary, + // use the unaligned copy version. + eor r3, r0, r1 + ands r3, r3, #0x3 + bne .L_copy_unknown_alignment +END(MEMCPY_BASE) + +ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED) + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + + // To try and improve performance, stack layout changed, + // i.e., not keeping the stack looking like users expect + // (highest numbered register at highest address). + strd r4, r5, [sp, #-8]! + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 + strd r6, r7, [sp, #-8]! + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r6, 0 + .cfi_rel_offset r7, 0 + strd r8, r9, [sp, #-8]! + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r8, 0 + .cfi_rel_offset r9, 4 + + // Optimized for already aligned dst code. + ands ip, r0, #3 + bne .L_dst_not_word_aligned + +.L_word_aligned: + // Align the destination buffer to 8 bytes, to make sure double + // loads and stores don't cross a cache line boundary, + // as they are then more expensive even if the data is in the cache + // (require two load/store issue cycles instead of one). + // If only one of the buffers is not 8 bytes aligned, + // then it's more important to align dst than src, + // because there is more penalty for stores + // than loads that cross a cacheline boundary. + // This check and realignment are only done if there is >= 832 + // bytes to copy. + + // Dst is word aligned, but check if it is already double word aligned. + ands r3, r0, #4 + beq 1f + ldr r3, [r1], #4 + str r3, [r0], #4 + sub r2, #4 + +1: // Can only get here if > 64 bytes to copy, so don't do check r2. + sub r2, #64 + +2: // Every loop iteration copies 64 bytes. + .irp offset, #0, #8, #16, #24, #32 + ldrd r4, r5, [r1, \offset] + strd r4, r5, [r0, \offset] + .endr + + ldrd r4, r5, [r1, #40] + ldrd r6, r7, [r1, #48] + ldrd r8, r9, [r1, #56] + + // Keep the pld as far from the next load as possible. + // The amount to prefetch was determined experimentally using + // large sizes, and verifying the prefetch size does not affect + // the smaller copies too much. + // WARNING: If the ldrd and strd instructions get too far away + // from each other, performance suffers. Three loads + // in a row is the best tradeoff. + pld [r1, #(64*16)] + strd r4, r5, [r0, #40] + strd r6, r7, [r0, #48] + strd r8, r9, [r0, #56] + + add r0, r0, #64 + add r1, r1, #64 + subs r2, r2, #64 + bge 2b + + // Fix-up the remaining count and make sure we have >= 32 bytes left. + adds r2, r2, #32 + blo 4f + + // Copy 32 bytes. These cache lines were already preloaded. + .irp offset, #0, #8, #16, #24 + ldrd r4, r5, [r1, \offset] + strd r4, r5, [r0, \offset] + .endr + add r1, r1, #32 + add r0, r0, #32 + sub r2, r2, #32 +4: // Less than 32 left. + add r2, r2, #32 + tst r2, #0x10 + beq 5f + // Copy 16 bytes. + .irp offset, #0, #8 + ldrd r4, r5, [r1, \offset] + strd r4, r5, [r0, \offset] + .endr + add r1, r1, #16 + add r0, r0, #16 + +5: // Copy up to 15 bytes (count in r2). + movs ip, r2, lsl #29 + bcc 1f + // Copy 8 bytes. + ldrd r4, r5, [r1], #8 + strd r4, r5, [r0], #8 +1: bge 2f + // Copy 4 bytes. + ldr r4, [r1], #4 + str r4, [r0], #4 +2: // Copy 0 to 4 bytes. + lsls r2, r2, #31 + itt ne + ldrbne lr, [r1], #1 + strbne lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1] + strbcs ip, [r0], #1 + strbcs lr, [r0] + + // Restore registers: optimized pop {r0, pc} + ldrd r8, r9, [sp], #8 + ldrd r6, r7, [sp], #8 + ldrd r4, r5, [sp], #8 + pop {r0, pc} + +.L_dst_not_word_aligned: + // Align dst to word. + rsb ip, ip, #4 + cmp ip, #2 + + itt gt + ldrbgt lr, [r1], #1 + strbgt lr, [r0], #1 + + itt ge + ldrbge lr, [r1], #1 + strbge lr, [r0], #1 + + ldrb lr, [r1], #1 + strb lr, [r0], #1 + + sub r2, r2, ip + + // Src is guaranteed to be at least word aligned by this point. + b .L_word_aligned +END(MEMCPY_BASE_ALIGNED) diff --git a/libc/arch-arm/cortex-a7/bionic/memset.S b/libc/arch-arm/cortex-a7/bionic/memset.S new file mode 100644 index 000000000..6365b066a --- /dev/null +++ b/libc/arch-arm/cortex-a7/bionic/memset.S @@ -0,0 +1,180 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include + + /* + * Optimized memset() for ARM. + * + * memset() returns its first argument. + */ + + .fpu neon + .syntax unified + +ENTRY(__memset_chk) + cmp r2, r3 + bls .L_done + + // Preserve lr for backtrace. + push {lr} + .cfi_def_cfa_offset 4 + .cfi_rel_offset lr, 0 + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMSET_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) +END(__memset_chk) + +ENTRY(bzero) + mov r2, r1 + mov r1, #0 +.L_done: + // Fall through to memset... +END(bzero) + +ENTRY(memset) + mov r3, r0 + // At this point only d0, d1 are going to be used below. + vdup.8 q0, r1 + cmp r2, #16 + blo .L_set_less_than_16_unknown_align + +.L_check_alignment: + // Align destination to a double word to avoid the store crossing + // a cache line boundary. + ands ip, r3, #7 + bne .L_do_double_word_align + +.L_double_word_aligned: + // Duplicate since the less than 64 can use d2, d3. + vmov q1, q0 + subs r2, #64 + blo .L_set_less_than_64 + + // Duplicate the copy value so that we can store 64 bytes at a time. + vmov q2, q0 + vmov q3, q0 + +1: // Main loop stores 64 bytes at a time. + subs r2, #64 + vstmia r3!, {d0 - d7} + bge 1b + +.L_set_less_than_64: + // Restore r2 to the count of bytes left to set. + add r2, #64 + lsls ip, r2, #27 + bcc .L_set_less_than_32 + // Set 32 bytes. + vstmia r3!, {d0 - d3} + +.L_set_less_than_32: + bpl .L_set_less_than_16 + // Set 16 bytes. + vstmia r3!, {d0, d1} + +.L_set_less_than_16: + // Less than 16 bytes to set. + lsls ip, r2, #29 + bcc .L_set_less_than_8 + + // Set 8 bytes. + vstmia r3!, {d0} + +.L_set_less_than_8: + bpl .L_set_less_than_4 + // Set 4 bytes + vst1.32 {d0[0]}, [r3]! + +.L_set_less_than_4: + lsls ip, r2, #31 + it ne + strbne r1, [r3], #1 + itt cs + strbcs r1, [r3], #1 + strbcs r1, [r3] + bx lr + +.L_do_double_word_align: + rsb ip, ip, #8 + sub r2, r2, ip + + // Do this comparison now, otherwise we'll need to save a + // register to the stack since we've used all available + // registers. + cmp ip, #4 + blo 1f + + // Need to do a four byte copy. + movs ip, ip, lsl #31 + it mi + strbmi r1, [r3], #1 + itt cs + strbcs r1, [r3], #1 + strbcs r1, [r3], #1 + vst1.32 {d0[0]}, [r3]! + b .L_double_word_aligned + +1: + // No four byte copy. + movs ip, ip, lsl #31 + it mi + strbmi r1, [r3], #1 + itt cs + strbcs r1, [r3], #1 + strbcs r1, [r3], #1 + b .L_double_word_aligned + +.L_set_less_than_16_unknown_align: + // Set up to 15 bytes. + movs ip, r2, lsl #29 + bcc 1f + vst1.8 {d0}, [r3]! +1: bge 2f + vst1.32 {d0[0]}, [r3]! +2: movs ip, r2, lsl #31 + it mi + strbmi r1, [r3], #1 + itt cs + strbcs r1, [r3], #1 + strbcs r1, [r3], #1 + bx lr +END(memset) + + .data +error_string: + .string "memset: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a7/cortex-a7.mk b/libc/arch-arm/cortex-a7/cortex-a7.mk index 9af03d94b..646c8b612 100644 --- a/libc/arch-arm/cortex-a7/cortex-a7.mk +++ b/libc/arch-arm/cortex-a7/cortex-a7.mk @@ -1 +1,18 @@ -include bionic/libc/arch-arm/cortex-a15/cortex-a15.mk +libc_bionic_src_files_arm += \ + arch-arm/cortex-a7/bionic/memset.S \ + +libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memchr.S \ + arch-arm/cortex-a15/bionic/memcpy.S \ + arch-arm/cortex-a15/bionic/strcat.S \ + arch-arm/cortex-a15/bionic/__strcat_chk.S \ + arch-arm/cortex-a15/bionic/strcmp.S \ + arch-arm/cortex-a15/bionic/strcpy.S \ + arch-arm/cortex-a15/bionic/__strcpy_chk.S \ + arch-arm/cortex-a15/bionic/strlen.S \ + +libc_bionic_src_files_arm += \ + arch-arm/bionic/memcmp.S \ + +libc_bionic_src_files_arm += \ + arch-arm/denver/bionic/memmove.S \ diff --git a/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S index 45517f146..0ece9630f 100644 --- a/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S +++ b/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S @@ -65,7 +65,7 @@ ENTRY(__strcat_chk) .p2align 2 .L_mainloop: - ldmia r1!, {r2, r3} + ldrd r2, r3, [r1], #8 pld [r1, #64] diff --git a/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S index 67eca086e..562b68873 100644 --- a/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S +++ b/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S @@ -51,7 +51,7 @@ ENTRY(__strcpy_chk) .p2align 2 .L_mainloop: - ldmia r0!, {r2, r3} + ldrd r2, r3, [r0], #8 pld [r0, #64] diff --git a/libc/arch-arm/cortex-a9/bionic/memcpy_base.S b/libc/arch-arm/cortex-a9/bionic/memcpy_base.S index 5e813050a..966b9b3c0 100644 --- a/libc/arch-arm/cortex-a9/bionic/memcpy_base.S +++ b/libc/arch-arm/cortex-a9/bionic/memcpy_base.S @@ -44,7 +44,7 @@ ENTRY_PRIVATE(MEMCPY_BASE) /* check if buffers are aligned. If so, run arm-only version */ eor r3, r0, r1 ands r3, r3, #0x3 - beq __memcpy_base_aligned + beq MEMCPY_BASE_ALIGNED /* Check the upper size limit for Neon unaligned memory access in memcpy */ cmp r2, #224 @@ -133,8 +133,7 @@ ENTRY_PRIVATE(MEMCPY_BASE) strbcs ip, [r0], #1 strbcs lr, [r0], #1 - ldmfd sp!, {r0, lr} - bx lr + ldmfd sp!, {r0, pc} END(MEMCPY_BASE) ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED) diff --git a/libc/arch-arm/cortex-a9/bionic/memset.S b/libc/arch-arm/cortex-a9/bionic/memset.S index 299f5a2d2..b39fcc4ae 100644 --- a/libc/arch-arm/cortex-a9/bionic/memset.S +++ b/libc/arch-arm/cortex-a9/bionic/memset.S @@ -35,6 +35,7 @@ */ .fpu neon + .syntax unified ENTRY(__memset_chk) cmp r2, r3 @@ -68,12 +69,9 @@ END(bzero) ENTRY(memset) // The neon memset only wins for less than 132. cmp r2, #132 - bhi __memset_large_copy - - stmfd sp!, {r0} - .cfi_def_cfa_offset 4 - .cfi_rel_offset r0, 0 + bhi .L_memset_large_copy + mov r3, r0 vdup.8 q0, r1 /* make sure we have at least 32 bytes to write */ @@ -83,7 +81,7 @@ ENTRY(memset) 1: /* The main loop writes 32 bytes at a time */ subs r2, r2, #32 - vst1.8 {d0 - d3}, [r0]! + vst1.8 {d0 - d3}, [r3]! bhs 1b 2: /* less than 32 left */ @@ -92,22 +90,20 @@ ENTRY(memset) beq 3f // writes 16 bytes, 128-bits aligned - vst1.8 {d0, d1}, [r0]! + vst1.8 {d0, d1}, [r3]! 3: /* write up to 15-bytes (count in r2) */ movs ip, r2, lsl #29 bcc 1f - vst1.8 {d0}, [r0]! + vst1.8 {d0}, [r3]! 1: bge 2f - vst1.32 {d0[0]}, [r0]! + vst1.32 {d0[0]}, [r3]! 2: movs ip, r2, lsl #31 - strmib r1, [r0], #1 - strcsb r1, [r0], #1 - strcsb r1, [r0], #1 - ldmfd sp!, {r0} + strbmi r1, [r3], #1 + strbcs r1, [r3], #1 + strbcs r1, [r3], #1 bx lr -END(memset) -ENTRY_PRIVATE(__memset_large_copy) +.L_memset_large_copy: /* compute the offset to align the destination * offset = (4-(src&3))&3 = -src & 3 */ @@ -131,12 +127,11 @@ ENTRY_PRIVATE(__memset_large_copy) orr r1, r1, r1, lsr #16 movs r12, r3, lsl #31 - strcsb r1, [r0], #1 /* can't use strh (alignment unknown) */ - strcsb r1, [r0], #1 - strmib r1, [r0], #1 + strbcs r1, [r0], #1 /* can't use strh (alignment unknown) */ + strbcs r1, [r0], #1 + strbmi r1, [r0], #1 subs r2, r2, r3 - ldmlsfd sp!, {r0, r4-r7, lr} /* return */ - bxls lr + popls {r0, r4-r7, pc} /* return */ /* align the destination to a cache-line */ mov r12, r1 @@ -155,9 +150,9 @@ ENTRY_PRIVATE(__memset_large_copy) /* conditionally writes 0 to 7 words (length in r3) */ movs r3, r3, lsl #28 - stmcsia r0!, {r1, lr} - stmcsia r0!, {r1, lr} - stmmiia r0!, {r1, lr} + stmcs r0!, {r1, lr} + stmcs r0!, {r1, lr} + stmmi r0!, {r1, lr} movs r3, r3, lsl #2 strcs r1, [r0], #4 @@ -172,16 +167,15 @@ ENTRY_PRIVATE(__memset_large_copy) /* conditionally stores 0 to 31 bytes */ movs r2, r2, lsl #28 - stmcsia r0!, {r1,r3,r12,lr} - stmmiia r0!, {r1, lr} + stmcs r0!, {r1,r3,r12,lr} + stmmi r0!, {r1, lr} movs r2, r2, lsl #2 strcs r1, [r0], #4 - strmih r1, [r0], #2 + strhmi r1, [r0], #2 movs r2, r2, lsl #2 - strcsb r1, [r0] - ldmfd sp!, {r0, r4-r7, lr} - bx lr -END(__memset_large_copy) + strbcs r1, [r0] + ldmfd sp!, {r0, r4-r7, pc} +END(memset) .data error_string: diff --git a/libc/arch-arm/cortex-a9/bionic/strcat.S b/libc/arch-arm/cortex-a9/bionic/strcat.S index f5a855e39..6f7203284 100644 --- a/libc/arch-arm/cortex-a9/bionic/strcat.S +++ b/libc/arch-arm/cortex-a9/bionic/strcat.S @@ -164,7 +164,7 @@ strcpy_check_src_align: .p2align 2 strcpy_mainloop: - ldmia r1!, {r2, r3} + ldrd r2, r3, [r1], #8 pld [r1, #64] diff --git a/libc/arch-arm/cortex-a9/bionic/string_copy.S b/libc/arch-arm/cortex-a9/bionic/string_copy.S index caf5a11fe..f691a08cb 100644 --- a/libc/arch-arm/cortex-a9/bionic/string_copy.S +++ b/libc/arch-arm/cortex-a9/bionic/string_copy.S @@ -134,7 +134,7 @@ ENTRY(strcpy) .p2align 2 .Lstringcopy_mainloop: - ldmia r1!, {r2, r3} + ldrd r2, r3, [r1], #8 pld [r1, #64] diff --git a/libc/arch-arm/cortex-a9/bionic/strlen.S b/libc/arch-arm/cortex-a9/bionic/strlen.S index b92b04352..271bc29f4 100644 --- a/libc/arch-arm/cortex-a9/bionic/strlen.S +++ b/libc/arch-arm/cortex-a9/bionic/strlen.S @@ -69,7 +69,7 @@ ENTRY(strlen) .p2align 2 mainloop: - ldmia r1!, {r2, r3} + ldrd r2, r3, [r1], #8 pld [r1, #64] diff --git a/libc/arch-arm/cortex-a9/cortex-a9.mk b/libc/arch-arm/cortex-a9/cortex-a9.mk index c82db3b4d..af1e7cfa9 100644 --- a/libc/arch-arm/cortex-a9/cortex-a9.mk +++ b/libc/arch-arm/cortex-a9/cortex-a9.mk @@ -1,4 +1,5 @@ libc_bionic_src_files_arm += \ + bionic/memchr.c \ arch-arm/cortex-a9/bionic/memcpy.S \ arch-arm/cortex-a9/bionic/memset.S \ arch-arm/cortex-a9/bionic/stpcpy.S \ @@ -9,3 +10,6 @@ libc_bionic_src_files_arm += \ arch-arm/cortex-a9/bionic/__strcpy_chk.S \ arch-arm/cortex-a9/bionic/strlen.S \ bionic/memmove.c \ + +libc_common_cflags_arm += \ + -DHAVE_32_BYTE_CACHE_LINE diff --git a/libc/arch-arm/denver/bionic/memset.S b/libc/arch-arm/denver/bionic/memset.S index bf3d9aded..d77c2443b 100644 --- a/libc/arch-arm/denver/bionic/memset.S +++ b/libc/arch-arm/denver/bionic/memset.S @@ -37,6 +37,7 @@ * memset() returns its first argument. */ + .cpu cortex-a15 .fpu neon .syntax unified diff --git a/libc/arch-arm/denver/denver.mk b/libc/arch-arm/denver/denver.mk index 0bc52a27b..61c55a193 100644 --- a/libc/arch-arm/denver/denver.mk +++ b/libc/arch-arm/denver/denver.mk @@ -5,8 +5,9 @@ libc_bionic_src_files_arm += \ arch-arm/denver/bionic/__strcat_chk.S \ arch-arm/denver/bionic/__strcpy_chk.S \ -# Use cortex-a15 versions of strcat/strcpy/strlen. +# Use cortex-a15 versions of memchr/strcat/strcpy/strlen. libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memchr.S \ arch-arm/cortex-a15/bionic/stpcpy.S \ arch-arm/cortex-a15/bionic/strcat.S \ arch-arm/cortex-a15/bionic/strcmp.S \ diff --git a/libc/arch-arm/generic/bionic/memcpy.S b/libc/arch-arm/generic/bionic/memcpy.S index b0c79abf7..65cba4c5e 100644 --- a/libc/arch-arm/generic/bionic/memcpy.S +++ b/libc/arch-arm/generic/bionic/memcpy.S @@ -37,6 +37,8 @@ * so we have to preserve R0. */ + .syntax unified + ENTRY(__memcpy_chk) cmp r2, r3 bhi __memcpy_chk_fail @@ -81,12 +83,12 @@ ENTRY(memcpy) */ movs r12, r3, lsl #31 sub r2, r2, r3 /* we know that r3 <= r2 because r2 >= 4 */ - ldrmib r3, [r1], #1 - ldrcsb r4, [r1], #1 - ldrcsb r12,[r1], #1 - strmib r3, [r0], #1 - strcsb r4, [r0], #1 - strcsb r12,[r0], #1 + ldrbmi r3, [r1], #1 + ldrbcs r4, [r1], #1 + ldrbcs r12,[r1], #1 + strbmi r3, [r0], #1 + strbcs r4, [r0], #1 + strbcs r12,[r0], #1 .Lsrc_aligned: @@ -109,10 +111,10 @@ ENTRY(memcpy) /* conditionally copies 0 to 7 words (length in r3) */ movs r12, r3, lsl #28 - ldmcsia r1!, {r4, r5, r6, r7} /* 16 bytes */ - ldmmiia r1!, {r8, r9} /* 8 bytes */ - stmcsia r0!, {r4, r5, r6, r7} - stmmiia r0!, {r8, r9} + ldmcs r1!, {r4, r5, r6, r7} /* 16 bytes */ + ldmmi r1!, {r8, r9} /* 8 bytes */ + stmcs r0!, {r4, r5, r6, r7} + stmmi r0!, {r8, r9} tst r3, #0x4 ldrne r10,[r1], #4 /* 4 bytes */ strne r10,[r0], #4 @@ -177,23 +179,22 @@ ENTRY(memcpy) /* conditionnaly copies 0 to 31 bytes */ movs r12, r2, lsl #28 - ldmcsia r1!, {r4, r5, r6, r7} /* 16 bytes */ - ldmmiia r1!, {r8, r9} /* 8 bytes */ - stmcsia r0!, {r4, r5, r6, r7} - stmmiia r0!, {r8, r9} + ldmcs r1!, {r4, r5, r6, r7} /* 16 bytes */ + ldmmi r1!, {r8, r9} /* 8 bytes */ + stmcs r0!, {r4, r5, r6, r7} + stmmi r0!, {r8, r9} movs r12, r2, lsl #30 ldrcs r3, [r1], #4 /* 4 bytes */ - ldrmih r4, [r1], #2 /* 2 bytes */ + ldrhmi r4, [r1], #2 /* 2 bytes */ strcs r3, [r0], #4 - strmih r4, [r0], #2 + strhmi r4, [r0], #2 tst r2, #0x1 - ldrneb r3, [r1] /* last byte */ - strneb r3, [r0] + ldrbne r3, [r1] /* last byte */ + strbne r3, [r0] /* we're done! restore everything and return */ 1: ldmfd sp!, {r5-r11} - ldmfd sp!, {r0, r4, lr} - bx lr + ldmfd sp!, {r0, r4, pc} /********************************************************************/ @@ -228,11 +229,11 @@ ENTRY(memcpy) * becomes aligned to 32 bits (r5 = nb of words to copy for alignment) */ movs r5, r5, lsl #31 - strmib r3, [r0], #1 + strbmi r3, [r0], #1 movmi r3, r3, lsr #8 - strcsb r3, [r0], #1 + strbcs r3, [r0], #1 movcs r3, r3, lsr #8 - strcsb r3, [r0], #1 + strbcs r3, [r0], #1 movcs r3, r3, lsr #8 cmp r2, #4 @@ -363,28 +364,27 @@ ENTRY(memcpy) .Lpartial_word_tail: /* we have a partial word in the input buffer */ movs r5, lr, lsl #(31-3) - strmib r3, [r0], #1 + strbmi r3, [r0], #1 movmi r3, r3, lsr #8 - strcsb r3, [r0], #1 + strbcs r3, [r0], #1 movcs r3, r3, lsr #8 - strcsb r3, [r0], #1 + strbcs r3, [r0], #1 /* Refill spilled registers from the stack. Don't update sp. */ ldmfd sp, {r5-r11} .Lcopy_last_3_and_return: movs r2, r2, lsl #31 /* copy remaining 0, 1, 2 or 3 bytes */ - ldrmib r2, [r1], #1 - ldrcsb r3, [r1], #1 - ldrcsb r12,[r1] - strmib r2, [r0], #1 - strcsb r3, [r0], #1 - strcsb r12,[r0] + ldrbmi r2, [r1], #1 + ldrbcs r3, [r1], #1 + ldrbcs r12,[r1] + strbmi r2, [r0], #1 + strbcs r3, [r0], #1 + strbcs r12,[r0] /* we're done! restore sp and spilled registers and return */ add sp, sp, #28 - ldmfd sp!, {r0, r4, lr} - bx lr + ldmfd sp!, {r0, r4, pc} END(memcpy) // Only reached when the __memcpy_chk check fails. diff --git a/libc/arch-arm/generic/bionic/memset.S b/libc/arch-arm/generic/bionic/memset.S index be35de9ff..b8eabbf83 100644 --- a/libc/arch-arm/generic/bionic/memset.S +++ b/libc/arch-arm/generic/bionic/memset.S @@ -35,6 +35,8 @@ * memset() returns its first argument. */ + .syntax unified + ENTRY(__memset_chk) cmp r2, r3 bls done @@ -76,12 +78,11 @@ ENTRY(memset) orr r1, r1, r1, lsr #16 movs r12, r3, lsl #31 - strcsb r1, [r0], #1 /* can't use strh (alignment unknown) */ - strcsb r1, [r0], #1 - strmib r1, [r0], #1 + strbcs r1, [r0], #1 /* can't use strh (alignment unknown) */ + strbcs r1, [r0], #1 + strbmi r1, [r0], #1 subs r2, r2, r3 - ldmlsfd sp!, {r0, r4-r7, lr} /* return */ - bxls lr + popls {r0, r4-r7, pc} /* return */ /* align the destination to a cache-line */ mov r12, r1 @@ -100,9 +101,9 @@ ENTRY(memset) /* conditionally writes 0 to 7 words (length in r3) */ movs r3, r3, lsl #28 - stmcsia r0!, {r1, lr} - stmcsia r0!, {r1, lr} - stmmiia r0!, {r1, lr} + stmcs r0!, {r1, lr} + stmcs r0!, {r1, lr} + stmmi r0!, {r1, lr} movs r3, r3, lsl #2 strcs r1, [r0], #4 @@ -117,15 +118,14 @@ ENTRY(memset) /* conditionally stores 0 to 31 bytes */ movs r2, r2, lsl #28 - stmcsia r0!, {r1,r3,r12,lr} - stmmiia r0!, {r1, lr} + stmcs r0!, {r1,r3,r12,lr} + stmmi r0!, {r1, lr} movs r2, r2, lsl #2 strcs r1, [r0], #4 - strmih r1, [r0], #2 + strhmi r1, [r0], #2 movs r2, r2, lsl #2 - strcsb r1, [r0] - ldmfd sp!, {r0, r4-r7, lr} - bx lr + strbcs r1, [r0] + ldmfd sp!, {r0, r4-r7, pc} END(memset) .data diff --git a/libc/arch-arm/generic/bionic/strcpy.S b/libc/arch-arm/generic/bionic/strcpy.S index 802a62da8..89ea098ad 100644 --- a/libc/arch-arm/generic/bionic/strcpy.S +++ b/libc/arch-arm/generic/bionic/strcpy.S @@ -32,6 +32,8 @@ #include #include +.syntax unified + ENTRY(strcpy) pld [r1, #0] eor r2, r0, r1 @@ -108,15 +110,15 @@ ENTRY(strcpy) #ifdef __ARMEB__ tst r2, #0xff00 iteet ne - strneh r2, [ip], #2 + strhne r2, [ip], #2 lsreq r2, r2, #8 - streqb r2, [ip] + strbeq r2, [ip] tstne r2, #0xff #else tst r2, #0xff itet ne - strneh r2, [ip], #2 - streqb r2, [ip] + strhne r2, [ip], #2 + strbeq r2, [ip] tstne r2, #0xff00 #endif bne 5b diff --git a/libc/arch-arm/generic/generic.mk b/libc/arch-arm/generic/generic.mk index 95be867cd..760c5bc78 100644 --- a/libc/arch-arm/generic/generic.mk +++ b/libc/arch-arm/generic/generic.mk @@ -1,4 +1,5 @@ libc_bionic_src_files_arm += \ + bionic/memchr.c \ arch-arm/generic/bionic/memcpy.S \ arch-arm/generic/bionic/memset.S \ arch-arm/generic/bionic/strcmp.S \ @@ -7,5 +8,4 @@ libc_bionic_src_files_arm += \ bionic/memmove.c \ bionic/__strcat_chk.cpp \ bionic/__strcpy_chk.cpp \ - upstream-openbsd/lib/libc/string/stpcpy.c \ upstream-openbsd/lib/libc/string/strcat.c \ diff --git a/libc/arch-arm/krait/bionic/memcpy_base.S b/libc/arch-arm/krait/bionic/memcpy_base.S index 035dcf126..068f2f60c 100644 --- a/libc/arch-arm/krait/bionic/memcpy_base.S +++ b/libc/arch-arm/krait/bionic/memcpy_base.S @@ -1,123 +1,215 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ +/*************************************************************************** + Copyright (c) 2009-2013 The Linux Foundation. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of The Linux Foundation nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + ***************************************************************************/ + +/* Assumes neon instructions and a cache line size of 64 bytes. */ +#include +#include /* - * This code assumes it is running on a processor that supports all arm v7 - * instructions, that supports neon instructions, and that has a 32 byte - * cache line. + * These default settings are good for all Krait-based systems + * as of this writing, but they can be overridden in: + * device///BoardConfig.mk + * by setting the following: + * TARGET_USE_KRAIT_BIONIC_OPTIMIZATION := true + * TARGET_USE_KRAIT_PLD_SET := true + * TARGET_KRAIT_BIONIC_PLDOFFS := + * TARGET_KRAIT_BIONIC_PLDSIZE := + * TARGET_KRAIT_BIONIC_PLDTHRESH := + * TARGET_KRAIT_BIONIC_BBTHRESH := */ -// Assumes neon instructions and a cache line size of 32 bytes. - -ENTRY_PRIVATE(MEMCPY_BASE) - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - - /* do we have at least 16-bytes to copy (needed for alignment below) */ - cmp r2, #16 - blo 5f - - /* align destination to cache-line for the write-buffer */ - rsb r3, r0, #0 - ands r3, r3, #0xF - beq 2f - - /* copy up to 15-bytes (count in r3) */ - sub r2, r2, r3 - movs ip, r3, lsl #31 - itt mi - ldrbmi lr, [r1], #1 - strbmi lr, [r0], #1 - itttt cs - ldrbcs ip, [r1], #1 - ldrbcs lr, [r1], #1 - strbcs ip, [r0], #1 - strbcs lr, [r0], #1 - movs ip, r3, lsl #29 - bge 1f - // copies 4 bytes, destination 32-bits aligned - vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! - vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! -1: bcc 2f - // copies 8 bytes, destination 64-bits aligned - vld1.8 {d0}, [r1]! - vst1.8 {d0}, [r0, :64]! - -2: /* make sure we have at least 64 bytes to copy */ - subs r2, r2, #64 - blo 2f - -1: /* The main loop copies 64 bytes at a time */ - vld1.8 {d0 - d3}, [r1]! - vld1.8 {d4 - d7}, [r1]! - pld [r1, #(32*8)] - subs r2, r2, #64 - vst1.8 {d0 - d3}, [r0, :128]! - vst1.8 {d4 - d7}, [r0, :128]! - bhs 1b - -2: /* fix-up the remaining count and make sure we have >= 32 bytes left */ - adds r2, r2, #32 - blo 4f - - /* Copy 32 bytes. These cache lines were already preloaded */ - vld1.8 {d0 - d3}, [r1]! - sub r2, r2, #32 - vst1.8 {d0 - d3}, [r0, :128]! - -4: /* less than 32 left */ - add r2, r2, #32 - tst r2, #0x10 - beq 5f - // copies 16 bytes, 128-bits aligned - vld1.8 {d0, d1}, [r1]! - vst1.8 {d0, d1}, [r0, :128]! - -5: /* copy up to 15-bytes (count in r2) */ - movs ip, r2, lsl #29 - bcc 1f - vld1.8 {d0}, [r1]! - vst1.8 {d0}, [r0]! -1: bge 2f - vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! - vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! -2: movs ip, r2, lsl #31 - itt mi - ldrbmi r3, [r1], #1 - strbmi r3, [r0], #1 - itttt cs - ldrbcs ip, [r1], #1 - ldrbcs lr, [r1], #1 - strbcs ip, [r0], #1 - strbcs lr, [r0], #1 - - ldmfd sp!, {r0, lr} - bx lr +#ifndef PLDOFFS +#define PLDOFFS (10) +#endif +#ifndef PLDTHRESH +#define PLDTHRESH (PLDOFFS) +#endif +#ifndef BBTHRESH +#define BBTHRESH (4096/64) +#endif +#if (PLDOFFS < 1) +#error Routine does not support offsets less than 1 +#endif +#if (PLDTHRESH < PLDOFFS) +#error PLD threshold must be greater than or equal to the PLD offset +#endif +#ifndef PLDSIZE +#define PLDSIZE (64) +#endif + .text + .fpu neon + +ENTRY(MEMCPY_BASE) +MEMCPY_BASE_ALIGNED: + // .cfi_startproc + .save {r0, r9, r10, lr} + // .cfi_def_cfa_offset 8 + //.cfi_rel_offset r0, 0 + //.cfi_rel_offset lr, 4 + cmp r2, #4 + blt .Lneon_lt4 + cmp r2, #16 + blt .Lneon_lt16 + cmp r2, #32 + blt .Lneon_16 + cmp r2, #64 + blt .Lneon_copy_32_a + + mov r12, r2, lsr #6 + cmp r12, #PLDTHRESH + ble .Lneon_copy_64_loop_nopld + + push {r9, r10} + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r9, 0 + .cfi_rel_offset r10, 4 + + cmp r12, #BBTHRESH + ble .Lneon_prime_pump + + add lr, r0, #0x400 + add r9, r1, #(PLDOFFS*PLDSIZE) + sub lr, lr, r9 + lsl lr, lr, #21 + lsr lr, lr, #21 + add lr, lr, #(PLDOFFS*PLDSIZE) + cmp r12, lr, lsr #6 + ble .Lneon_prime_pump + + itt gt + movgt r9, #(PLDOFFS) + rsbsgt r9, r9, lr, lsr #6 + ble .Lneon_prime_pump + + add r10, r1, lr + bic r10, #0x3F + + sub r12, r12, lr, lsr #6 + + cmp r9, r12 + itee le + suble r12, r12, r9 + movgt r9, r12 + movgt r12, #0 + + pld [r1, #((PLDOFFS-1)*PLDSIZE)] +.Lneon_copy_64_loop_outer_doublepld: + pld [r1, #((PLDOFFS)*PLDSIZE)] + vld1.32 {q0, q1}, [r1]! + vld1.32 {q2, q3}, [r1]! + ldr r3, [r10] + subs r9, r9, #1 + vst1.32 {q0, q1}, [r0]! + vst1.32 {q2, q3}, [r0]! + add r10, #64 + bne .Lneon_copy_64_loop_outer_doublepld + cmp r12, #0 + beq .Lneon_pop_before_nopld + + cmp r12, #(512*1024/64) + blt .Lneon_copy_64_loop_outer + +.Lneon_copy_64_loop_ddr: + vld1.32 {q0, q1}, [r1]! + vld1.32 {q2, q3}, [r1]! + pld [r10] + subs r12, r12, #1 + vst1.32 {q0, q1}, [r0]! + vst1.32 {q2, q3}, [r0]! + add r10, #64 + bne .Lneon_copy_64_loop_ddr + b .Lneon_pop_before_nopld + +.Lneon_prime_pump: + mov lr, #(PLDOFFS*PLDSIZE) + add r10, r1, #(PLDOFFS*PLDSIZE) + bic r10, #0x3F + sub r12, r12, #PLDOFFS + ldr r3, [r10, #(-1*PLDSIZE)] +.Lneon_copy_64_loop_outer: + vld1.32 {q0, q1}, [r1]! + vld1.32 {q2, q3}, [r1]! + ldr r3, [r10] + subs r12, r12, #1 + vst1.32 {q0, q1}, [r0]! + vst1.32 {q2, q3}, [r0]! + add r10, #64 + bne .Lneon_copy_64_loop_outer +.Lneon_pop_before_nopld: + mov r12, lr, lsr #6 + pop {r9, r10} + .cfi_restore r9 + .cfi_restore r10 + .cfi_adjust_cfa_offset -8 + +.Lneon_copy_64_loop_nopld: + vld1.32 {q8, q9}, [r1]! + vld1.32 {q10, q11}, [r1]! + subs r12, r12, #1 + vst1.32 {q8, q9}, [r0]! + vst1.32 {q10, q11}, [r0]! + bne .Lneon_copy_64_loop_nopld + ands r2, r2, #0x3f + .cfi_restore r0 + .cfi_adjust_cfa_offset -4 + beq .Lneon_exit +.Lneon_copy_32_a: + movs r3, r2, lsl #27 + bcc .Lneon_16 + vld1.32 {q0,q1}, [r1]! + vst1.32 {q0,q1}, [r0]! +.Lneon_16: + bpl .Lneon_lt16 + vld1.32 {q8}, [r1]! + vst1.32 {q8}, [r0]! + ands r2, r2, #0x0f + beq .Lneon_exit +.Lneon_lt16: + movs r3, r2, lsl #29 + itttt cs + ldrcs r3, [r1], #4 + strcs r3, [r0], #4 + ldrcs r3, [r1], #4 + strcs r3, [r0], #4 + itt mi + ldrmi r3, [r1], #4 + strmi r3, [r0], #4 +.Lneon_lt4: + movs r2, r2, lsl #31 + itt cs + ldrhcs r3, [r1], #2 + strhcs r3, [r0], #2 + itt mi + ldrbmi r3, [r1] + strbmi r3, [r0] +.Lneon_exit: + pop {r0, lr} + bx lr + //.cfi_endproc END(MEMCPY_BASE) + diff --git a/libc/arch-arm/krait/bionic/memmove.S b/libc/arch-arm/krait/bionic/memmove.S new file mode 100644 index 000000000..698cc52ba --- /dev/null +++ b/libc/arch-arm/krait/bionic/memmove.S @@ -0,0 +1,219 @@ +/*************************************************************************** + Copyright (c) 2009-2014 The Linux Foundation. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of The Linux Foundation nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + ***************************************************************************/ + +/*************************************************************************** + * Neon memmove: Attempts to do a memmove with Neon registers if possible, + * Inputs: + * dest: The destination buffer + * src: The source buffer + * n: The size of the buffer to transfer + * Outputs: + * + ***************************************************************************/ + +#include +#include +/* + * These can be overridden in: + * device///BoardConfig.mk + * by setting the following: + * TARGET_USE_KRAIT_BIONIC_OPTIMIZATION := true + * TARGET_USE_KRAIT_PLD_SET := true + * TARGET_KRAIT_BIONIC_PLDOFFS := + * TARGET_KRAIT_BIONIC_PLDSIZE := + * TARGET_KRAIT_BIONIC_PLDTHRESH := + */ +#ifndef PLDOFFS +#define PLDOFFS (10) +#endif +#ifndef PLDTHRESH +#define PLDTHRESH (PLDOFFS) +#endif +#if (PLDOFFS < 5) +#error Routine does not support offsets less than 5 +#endif +#if (PLDTHRESH < PLDOFFS) +#error PLD threshold must be greater than or equal to the PLD offset +#endif +#ifndef PLDSIZE +#define PLDSIZE (64) +#endif + + .text + .syntax unified + .fpu neon + .thumb + .thumb_func + +ENTRY(bcopy) + //.cfi_startproc + mov r12, r0 + mov r0, r1 + mov r1, r12 + // Fall through to memmove + //.cfi_endproc +END(bcopy) + +ENTRY(memmove) +_memmove_words: + //.cfi_startproc + .save {r0, lr} + cmp r2, #0 + it ne + subsne r12, r0, r1 // Warning: do not combine these "it" blocks + it eq + bxeq lr +// memmove only if r1 < r0 < r1+r2 + cmp r0, r1 + itt ge + addge r12, r1, r2 + cmpge r12, r0 + it le + ble memcpy + cmp r2, #4 + it le + ble .Lneon_b2f_smallcopy_loop + push {r0, lr} + add r0, r0, r2 + add r1, r1, r2 + cmp r2, #64 + it ge + bge .Lneon_b2f_copy_64 + cmp r2, #32 + it ge + bge .Lneon_b2f_copy_32 + cmp r2, #8 + it ge + bge .Lneon_b2f_copy_8 + b .Lneon_b2f_copy_1 +.Lneon_b2f_copy_64: + mov r12, r2, lsr #6 + add r0, r0, #32 + add r1, r1, #32 + cmp r12, #PLDTHRESH + it le + ble .Lneon_b2f_copy_64_loop_nopld + sub r12, #PLDOFFS + sub lr, r1, #(PLDOFFS)*PLDSIZE +.Lneon_b2f_copy_64_loop_outer: + pld [lr] + sub r1, r1, #96 + sub r0, r0, #96 + vld1.32 {q0, q1}, [r1]! + vld1.32 {q2, q3}, [r1] + sub lr, lr, #64 + subs r12, r12, #1 + vst1.32 {q0, q1}, [r0]! + vst1.32 {q2, q3}, [r0] + it ne + bne .Lneon_b2f_copy_64_loop_outer + mov r12, #PLDOFFS +.Lneon_b2f_copy_64_loop_nopld: + sub r1, r1, #96 + sub r0, r0, #96 + vld1.32 {q8, q9}, [r1]! + vld1.32 {q10, q11}, [r1] + subs r12, r12, #1 + vst1.32 {q8, q9}, [r0]! + vst1.32 {q10, q11}, [r0] + it ne + bne .Lneon_b2f_copy_64_loop_nopld + ands r2, r2, #0x3f + it eq + beq .Lneon_memmove_done + sub r1, r1, #32 + sub r0, r0, #32 + cmp r2, #32 + it lt + blt .Lneon_b2f_copy_8 +.Lneon_b2f_copy_32: + sub r1, r1, #32 + sub r0, r0, #32 + vld1.32 {q0, q1}, [r1] + vst1.32 {q0, q1}, [r0] + ands r2, r2, #0x1f + it eq + beq .Lneon_memmove_done +.Lneon_b2f_copy_8: + movs r12, r2, lsr #0x3 + it eq + beq .Lneon_b2f_copy_1 +.Lneon_b2f_copy_8_loop: + sub r1, r1, #8 + sub r0, r0, #8 + vld1.32 {d0}, [r1] + subs r12, r12, #1 + vst1.32 {d0}, [r0] + it ne + bne .Lneon_b2f_copy_8_loop + ands r2, r2, #0x7 + beq .Lneon_memmove_done +.Lneon_b2f_copy_1: + movs r12, r2, lsl #29 + itttt mi + submi r1, r1, #4 + submi r0, r0, #4 + ldrmi r3, [r1] + strmi r3, [r0] + movs r2, r2, lsl #31 + itttt cs + subcs r1, r1, #2 + subcs r0, r0, #2 + ldrhcs r3, [r1] + strhcs r3, [r0] + itttt mi + submi r1, r1, #1 + submi r0, r0, #1 + ldrbmi r12, [r1] + strbmi r12, [r0] +.Lneon_memmove_done: + pop {r0, pc} +.Lneon_b2f_smallcopy_loop: + // 4 bytes or less + add r1, r1, r2 + add r0, r0, r2 + movs r12, r2, lsl #29 + itttt mi + submi r1, r1, #4 + submi r0, r0, #4 + ldrmi r3, [r1] + strmi r3, [r0] + movs r2, r2, lsl #31 + itttt cs + subcs r1, r1, #2 + subcs r0, r0, #2 + ldrhcs r3, [r1] + strhcs r3, [r0] + itttt mi + submi r1, r1, #1 + submi r0, r0, #1 + ldrbmi r12, [r1] + strbmi r12, [r0] + bx lr +// .cfi_endproc +END(memmove) + diff --git a/libc/arch-arm/krait/bionic/memset.S b/libc/arch-arm/krait/bionic/memset.S index e9f643133..ae05965f4 100644 --- a/libc/arch-arm/krait/bionic/memset.S +++ b/libc/arch-arm/krait/bionic/memset.S @@ -37,6 +37,7 @@ */ .fpu neon + .syntax unified ENTRY(__memset_chk) cmp r2, r3 @@ -68,10 +69,7 @@ END(bzero) /* memset() returns its first argument. */ ENTRY(memset) - stmfd sp!, {r0} - .cfi_def_cfa_offset 4 - .cfi_rel_offset r0, 0 - + mov r3, r0 vdup.8 q0, r1 /* make sure we have at least 32 bytes to write */ @@ -81,7 +79,7 @@ ENTRY(memset) 1: /* The main loop writes 32 bytes at a time */ subs r2, r2, #32 - vst1.8 {d0 - d3}, [r0]! + vst1.8 {d0 - d3}, [r3]! bhs 1b 2: /* less than 32 left */ @@ -90,18 +88,17 @@ ENTRY(memset) beq 3f // writes 16 bytes, 128-bits aligned - vst1.8 {d0, d1}, [r0]! + vst1.8 {d0, d1}, [r3]! 3: /* write up to 15-bytes (count in r2) */ movs ip, r2, lsl #29 bcc 1f - vst1.8 {d0}, [r0]! + vst1.8 {d0}, [r3]! 1: bge 2f - vst1.32 {d0[0]}, [r0]! + vst1.32 {d0[0]}, [r3]! 2: movs ip, r2, lsl #31 - strmib r1, [r0], #1 - strcsb r1, [r0], #1 - strcsb r1, [r0], #1 - ldmfd sp!, {r0} + strbmi r1, [r3], #1 + strbcs r1, [r3], #1 + strbcs r1, [r3], #1 bx lr END(memset) diff --git a/libc/arch-arm/krait/krait.mk b/libc/arch-arm/krait/krait.mk index 1bb7b0a07..8bf210035 100644 --- a/libc/arch-arm/krait/krait.mk +++ b/libc/arch-arm/krait/krait.mk @@ -1,9 +1,19 @@ libc_bionic_src_files_arm += \ - arch-arm/krait/bionic/memcpy.S \ + arch-arm/cortex-a15/bionic/memchr.S \ arch-arm/krait/bionic/memset.S \ arch-arm/krait/bionic/strcmp.S \ arch-arm/krait/bionic/__strcat_chk.S \ arch-arm/krait/bionic/__strcpy_chk.S \ + arch-arm/krait/bionic/memmove.S + +#For some targets we don't need this optimization +ifeq ($(TARGET_CPU_MEMCPY_BASE_OPT_DISABLE),true) +libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memcpy.S +else +libc_bionic_src_files_arm += \ + arch-arm/krait/bionic/memcpy.S +endif # Use cortex-a15 versions of strcat/strcpy/strlen and standard memmove libc_bionic_src_files_arm += \ @@ -11,4 +21,3 @@ libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/strcat.S \ arch-arm/cortex-a15/bionic/strcpy.S \ arch-arm/cortex-a15/bionic/strlen.S \ - bionic/memmove.c \ diff --git a/libc/arch-arm/scorpion/scorpion.mk b/libc/arch-arm/scorpion/scorpion.mk new file mode 100644 index 000000000..74abffd21 --- /dev/null +++ b/libc/arch-arm/scorpion/scorpion.mk @@ -0,0 +1,16 @@ +# Use krait versions of memset/strcmp/memmove +libc_bionic_src_files_arm += \ + arch-arm/krait/bionic/memset.S \ + arch-arm/krait/bionic/strcmp.S \ + arch-arm/krait/bionic/memmove.S + +# Use cortex-a15 versions of memcpy/strcat/strcpy/strlen +libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memchr.S \ + arch-arm/cortex-a15/bionic/memcpy.S \ + arch-arm/cortex-a15/bionic/stpcpy.S \ + arch-arm/cortex-a15/bionic/strcat.S \ + arch-arm/cortex-a15/bionic/strcpy.S \ + arch-arm/cortex-a15/bionic/strlen.S \ + arch-arm/cortex-a15/bionic/__strcat_chk.S \ + arch-arm/cortex-a15/bionic/__strcpy_chk.S \ diff --git a/libc/arch-arm/syscalls/ioprio_get.S b/libc/arch-arm/syscalls/ioprio_get.S new file mode 100644 index 000000000..19e04ed1f --- /dev/null +++ b/libc/arch-arm/syscalls/ioprio_get.S @@ -0,0 +1,14 @@ +/* Generated by gensyscalls.py. Do not edit. */ + +#include + +ENTRY(ioprio_get) + mov ip, r7 + ldr r7, =__NR_ioprio_get + swi #0 + mov r7, ip + cmn r0, #(MAX_ERRNO + 1) + bxls lr + neg r0, r0 + b __set_errno +END(ioprio_get) diff --git a/libc/arch-arm/syscalls/ioprio_set.S b/libc/arch-arm/syscalls/ioprio_set.S new file mode 100644 index 000000000..9b94efb59 --- /dev/null +++ b/libc/arch-arm/syscalls/ioprio_set.S @@ -0,0 +1,14 @@ +/* Generated by gensyscalls.py. Do not edit. */ + +#include + +ENTRY(ioprio_set) + mov ip, r7 + ldr r7, =__NR_ioprio_set + swi #0 + mov r7, ip + cmn r0, #(MAX_ERRNO + 1) + bxls lr + neg r0, r0 + b __set_errno +END(ioprio_set) diff --git a/libc/arch-arm64/arm64.mk b/libc/arch-arm64/arm64.mk index bb6ca6303..a4c32efe3 100644 --- a/libc/arch-arm64/arm64.mk +++ b/libc/arch-arm64/arm64.mk @@ -61,6 +61,6 @@ ifeq ($(wildcard $(cpu_variant_mk)),) $(error "TARGET_CPU_VARIANT not set or set to an unknown value. Possible values are generic, denver64. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.") endif include $(cpu_variant_mk) -libc_common_additional_dependencies += $(cpu_variank_mk) +libc_common_additional_dependencies += $(cpu_variant_mk) cpu_variant_mk := diff --git a/libc/arch-arm64/generic/bionic/memchr.S b/libc/arch-arm64/generic/bionic/memchr.S index e5ea57d8c..a00dd8dee 100644 --- a/libc/arch-arm64/generic/bionic/memchr.S +++ b/libc/arch-arm64/generic/bionic/memchr.S @@ -101,7 +101,7 @@ ENTRY(memchr) and vhas_chr2.16b, vhas_chr2.16b, vrepmask.16b addp vend.16b, vhas_chr1.16b, vhas_chr2.16b /* 256->128 */ addp vend.16b, vend.16b, vend.16b /* 128->64 */ - mov synd, vend.2d[0] + mov synd, vend.d[0] /* Clear the soff*2 lower bits */ lsl tmp, soff, #1 lsr synd, synd, tmp @@ -121,7 +121,7 @@ ENTRY(memchr) /* Use a fast check for the termination condition */ orr vend.16b, vhas_chr1.16b, vhas_chr2.16b addp vend.2d, vend.2d, vend.2d - mov synd, vend.2d[0] + mov synd, vend.d[0] /* We're not out of data, loop if we haven't found the character */ cbz synd, .Lloop @@ -131,7 +131,7 @@ ENTRY(memchr) and vhas_chr2.16b, vhas_chr2.16b, vrepmask.16b addp vend.16b, vhas_chr1.16b, vhas_chr2.16b /* 256->128 */ addp vend.16b, vend.16b, vend.16b /* 128->64 */ - mov synd, vend.2d[0] + mov synd, vend.d[0] /* Only do the clear for the last possible block */ b.hi .Ltail diff --git a/libc/arch-arm64/generic/bionic/memmove.S b/libc/arch-arm64/generic/bionic/memmove.S index 8b366a314..739ce4982 100644 --- a/libc/arch-arm64/generic/bionic/memmove.S +++ b/libc/arch-arm64/generic/bionic/memmove.S @@ -35,10 +35,6 @@ #include /* Parameters and result. */ -#ifdef BCOPY -#define origdstin x1 -#define origsrc x0 -#endif #define dstin x0 #define src x1 #define count x2 @@ -59,13 +55,7 @@ #define D_l x13 #define D_h x14 -#ifdef BCOPY -ENTRY(bcopy) - /* Swap src and dst so that a branch to memcpy doesn't cause issues. */ - mov tmp1, origsrc - mov origsrc, origdstin - mov origdstin, tmp1 -#elif defined(WMEMMOVE) +#if defined(WMEMMOVE) ENTRY(wmemmove) lsl count, count, #2 #else @@ -332,9 +322,7 @@ ENTRY(memmove) tst count, #0x3f b.ne .Ltail63down ret -#ifdef BCOPY -END(bcopy) -#elif defined(WMEMMOVE) +#if defined(WMEMMOVE) END(wmemmove) #else END(memmove) diff --git a/libc/arch-arm64/generic/bionic/strchr.S b/libc/arch-arm64/generic/bionic/strchr.S index 469b83c63..b54106d93 100644 --- a/libc/arch-arm64/generic/bionic/strchr.S +++ b/libc/arch-arm64/generic/bionic/strchr.S @@ -109,7 +109,7 @@ ENTRY(strchr) addp vend1.16b, vend1.16b, vend2.16b // 128->64 lsr tmp1, tmp3, tmp1 - mov tmp3, vend1.2d[0] + mov tmp3, vend1.d[0] bic tmp1, tmp3, tmp1 // Mask padding bits. cbnz tmp1, .Ltail @@ -124,7 +124,7 @@ ENTRY(strchr) orr vend2.16b, vhas_nul2.16b, vhas_chr2.16b orr vend1.16b, vend1.16b, vend2.16b addp vend1.2d, vend1.2d, vend1.2d - mov tmp1, vend1.2d[0] + mov tmp1, vend1.d[0] cbz tmp1, .Lloop /* Termination condition found. Now need to establish exactly why @@ -138,7 +138,7 @@ ENTRY(strchr) addp vend1.16b, vend1.16b, vend2.16b // 256->128 addp vend1.16b, vend1.16b, vend2.16b // 128->64 - mov tmp1, vend1.2d[0] + mov tmp1, vend1.d[0] .Ltail: /* Count the trailing zeros, by bit reversing... */ rbit tmp1, tmp1 diff --git a/libc/arch-arm64/generic/bionic/string_copy.S b/libc/arch-arm64/generic/bionic/string_copy.S index 2bf969d60..2691ca56a 100644 --- a/libc/arch-arm64/generic/bionic/string_copy.S +++ b/libc/arch-arm64/generic/bionic/string_copy.S @@ -1,34 +1,9 @@ /* - * Copyright (C) 2014 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ + strcpy/stpcpy - copy a string returning pointer to start/end. -/* - Copyright (c) 2014, Linaro Limited - All rights reserved. + Copyright (c) 2013, 2014, 2015 ARM Ltd. + All Rights Reserved. + Copyright (c) 2015, Linaro Ltd. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -37,9 +12,9 @@ * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of the Linaro nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. + * Neither the name of the company nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT @@ -51,12 +26,11 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Assumptions: * - * ARMv8-a, AArch64 + * ARMv8-a, AArch64, unaligned accesses, min page size 4k. */ #if !defined(STPCPY) && !defined(STRCPY) @@ -65,179 +39,298 @@ #include +/* To build as stpcpy, define STPCPY before compiling this file. + + To test the page crossing code path more thoroughly, compile with + -DSTRCPY_TEST_PAGE_CROSS - this will force all copies through the slower + entry path. This option is not intended for production use. */ + /* Arguments and results. */ -#if defined(STPCPY) -#define dst x0 -#elif defined(STRCPY) -#define dstin x0 -#endif -#define src x1 +#define dstin x0 +#define srcin x1 /* Locals and temporaries. */ -#if defined(STRCPY) -#define dst x2 -#endif -#define data1 x3 -#define data1_w w3 -#define data2 x4 -#define data2_w w4 -#define has_nul1 x5 -#define has_nul1_w w5 -#define has_nul2 x6 -#define tmp1 x7 -#define tmp2 x8 -#define tmp3 x9 -#define tmp4 x10 -#define zeroones x11 -#define zeroones_w w11 -#define pos x12 +#define src x2 +#define dst x3 +#define data1 x4 +#define data1w w4 +#define data2 x5 +#define data2w w5 +#define has_nul1 x6 +#define has_nul2 x7 +#define tmp1 x8 +#define tmp2 x9 +#define tmp3 x10 +#define tmp4 x11 +#define zeroones x12 +#define data1a x13 +#define data2a x14 +#define pos x15 +#define len x16 +#define to_align x17 + + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and + can be done in parallel across the entire word. */ #define REP8_01 0x0101010101010101 #define REP8_7f 0x7f7f7f7f7f7f7f7f #define REP8_80 0x8080808080808080 + /* AArch64 systems have a minimum page size of 4k. We can do a quick + page size check for crossing this boundary on entry and if we + do not, then we can short-circuit much of the entry code. We + expect early page-crossing strings to be rare (probability of + 16/MIN_PAGE_SIZE ~= 0.4%), so the branch should be quite + predictable, even with random strings. + + We don't bother checking for larger page sizes, the cost of setting + up the correct page size is just not worth the extra gain from + a small reduction in the cases taking the slow path. Note that + we only care about whether the first fetch, which may be + misaligned, crosses a page boundary - after that we move to aligned + fetches for the remainder of the string. */ + +#ifdef STRCPY_TEST_PAGE_CROSS + /* Make everything that isn't Qword aligned look like a page cross. */ +#define MIN_PAGE_P2 4 +#else +#define MIN_PAGE_P2 12 +#endif + +#define MIN_PAGE_SIZE (1 << MIN_PAGE_P2) + #if defined(STPCPY) ENTRY(stpcpy) #elif defined(STRCPY) ENTRY(strcpy) #endif - mov zeroones, #REP8_01 -#if defined(STRCPY) - mov dst, dstin + .p2align 6 + /* For moderately short strings, the fastest way to do the copy is to + calculate the length of the string in the same way as strlen, then + essentially do a memcpy of the result. This avoids the need for + multiple byte copies and further means that by the time we + reach the bulk copy loop we know we can always use DWord + accesses. We expect strcpy to rarely be called repeatedly + with the same source string, so branch prediction is likely to + always be difficult - we mitigate against this by preferring + conditional select operations over branches whenever this is + feasible. */ + and tmp2, srcin, #(MIN_PAGE_SIZE - 1) + mov zeroones, #REP8_01 + and to_align, srcin, #15 + cmp tmp2, #(MIN_PAGE_SIZE - 16) + neg tmp1, to_align + /* The first fetch will straddle a (possible) page boundary iff + srcin + 15 causes bit[MIN_PAGE_P2] to change value. A 16-byte + aligned string will never fail the page align check, so will + always take the fast path. */ + b.gt .Lpage_cross + +.Lpage_cross_ok: + ldp data1, data2, [srcin] +#ifdef __AARCH64EB__ + /* Because we expect the end to be found within 16 characters + (profiling shows this is the most common case), it's worth + swapping the bytes now to save having to recalculate the + termination syndrome later. We preserve data1 and data2 + so that we can re-use the values later on. */ + rev tmp2, data1 + sub tmp1, tmp2, zeroones + orr tmp2, tmp2, #REP8_7f + bics has_nul1, tmp1, tmp2 + b.ne .Lfp_le8 + rev tmp4, data2 + sub tmp3, tmp4, zeroones + orr tmp4, tmp4, #REP8_7f +#else + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + bics has_nul1, tmp1, tmp2 + b.ne .Lfp_le8 + sub tmp3, data2, zeroones + orr tmp4, data2, #REP8_7f #endif - ands tmp1, src, #15 - b.ne .Lmisaligned - // NUL detection works on the principle that (X - 1) & (~X) & 0x80 - // (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and - // can be done in parallel across the entire word. - // The inner loop deals with two Dwords at a time. This has a - // slightly higher start-up cost, but we should win quite quickly, - // especially on cores with a high number of issue slots per - // cycle, as we get much better parallelism out of the operations. -.Lloop: - ldp data1, data2, [src], #16 - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - bic has_nul1, tmp1, tmp2 - cbnz has_nul1, .Lnul_in_data1 - sub tmp3, data2, zeroones - orr tmp4, data2, #REP8_7f - bic has_nul2, tmp3, tmp4 - cbnz has_nul2, .Lnul_in_data2 - // No NUL in either register, copy it in a single instruction. - stp data1, data2, [dst], #16 - b .Lloop - -.Lnul_in_data1: - rev has_nul1, has_nul1 - clz pos, has_nul1 - add tmp1, pos, #0x8 - - tbz tmp1, #6, 1f -#if defined(STPCPY) - str data1, [dst], #7 -#elif defined(STRCPY) - str data1, [dst] + bics has_nul2, tmp3, tmp4 + b.eq .Lbulk_entry + + /* The string is short (<=16 bytes). We don't know exactly how + short though, yet. Work out the exact length so that we can + quickly select the optimal copy strategy. */ +.Lfp_gt8: + rev has_nul2, has_nul2 + clz pos, has_nul2 + mov tmp2, #56 + add dst, dstin, pos, lsr #3 /* Bits to bytes. */ + sub pos, tmp2, pos +#ifdef __AARCH64EB__ + lsr data2, data2, pos +#else + lsl data2, data2, pos #endif - ret -1: - tbz tmp1, #5, 1f - str data1_w, [dst], #4 - lsr data1, data1, #32 -1: - tbz tmp1, #4, 1f - strh data1_w, [dst], #2 - lsr data1, data1, #16 -1: - tbz tmp1, #3, 1f - strb data1_w, [dst] -#if defined(STPCPY) - ret + str data2, [dst, #1] + str data1, [dstin] +#ifdef STPCPY + add dstin, dst, #8 #endif -1: -#if defined(STPCPY) - // Back up one so that dst points to the '\0' string terminator. - sub dst, dst, #1 + ret + +.Lfp_le8: + rev has_nul1, has_nul1 + clz pos, has_nul1 + add dst, dstin, pos, lsr #3 /* Bits to bytes. */ + subs tmp2, pos, #24 /* Pos in bits. */ + b.lt .Lfp_lt4 +#ifdef __AARCH64EB__ + mov tmp2, #56 + sub pos, tmp2, pos + lsr data2, data1, pos + lsr data1, data1, #32 +#else + lsr data2, data1, tmp2 #endif - ret + /* 4->7 bytes to copy. */ + str data2w, [dst, #-3] + str data1w, [dstin] +#ifdef STPCPY + mov dstin, dst +#endif + ret +.Lfp_lt4: + cbz pos, .Lfp_lt2 + /* 2->3 bytes to copy. */ +#ifdef __AARCH64EB__ + lsr data1, data1, #48 +#endif + strh data1w, [dstin] + /* Fall-through, one byte (max) to go. */ +.Lfp_lt2: + /* Null-terminated string. Last character must be zero! */ + strb wzr, [dst] +#ifdef STPCPY + mov dstin, dst +#endif + ret -.Lnul_in_data2: - str data1, [dst], #8 - rev has_nul2, has_nul2 - clz pos, has_nul2 - add tmp1, pos, #0x8 + .p2align 6 + /* Aligning here ensures that the entry code and main loop all lies + within one 64-byte cache line. */ +.Lbulk_entry: + sub to_align, to_align, #16 + stp data1, data2, [dstin] + sub src, srcin, to_align + sub dst, dstin, to_align + b .Lentry_no_page_cross - tbz tmp1, #6, 1f -#if defined(STPCPY) - str data2, [dst], #7 -#elif defined(STRCPY) - str data2, [dst] -#endif - ret -1: - tbz tmp1, #5, 1f - str data2_w, [dst], #4 - lsr data2, data2, #32 -1: - tbz tmp1, #4, 1f - strh data2_w, [dst], #2 - lsr data2, data2, #16 -1: - tbz tmp1, #3, 1f - strb data2_w, [dst] -#if defined(STPCPY) - ret + /* The inner loop deals with two Dwords at a time. This has a + slightly higher start-up cost, but we should win quite quickly, + especially on cores with a high number of issue slots per + cycle, as we get much better parallelism out of the operations. */ +.Lmain_loop: + stp data1, data2, [dst], #16 +.Lentry_no_page_cross: + ldp data1, data2, [src], #16 + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + sub tmp3, data2, zeroones + orr tmp4, data2, #REP8_7f + bic has_nul1, tmp1, tmp2 + bics has_nul2, tmp3, tmp4 + ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */ + b.eq .Lmain_loop + + /* Since we know we are copying at least 16 bytes, the fastest way + to deal with the tail is to determine the location of the + trailing NUL, then (re)copy the 16 bytes leading up to that. */ + cmp has_nul1, #0 +#ifdef __AARCH64EB__ + /* For big-endian, carry propagation (if the final byte in the + string is 0x01) means we cannot use has_nul directly. The + easiest way to get the correct byte is to byte-swap the data + and calculate the syndrome a second time. */ + csel data1, data1, data2, ne + rev data1, data1 + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + bic has_nul1, tmp1, tmp2 +#else + csel has_nul1, has_nul1, has_nul2, ne #endif -1: -#if defined(STPCPY) - // Back up one so that dst points to the '\0' string terminator. - sub dst, dst, #1 + rev has_nul1, has_nul1 + clz pos, has_nul1 + add tmp1, pos, #72 + add pos, pos, #8 + csel pos, pos, tmp1, ne + add src, src, pos, lsr #3 + add dst, dst, pos, lsr #3 + ldp data1, data2, [src, #-32] + stp data1, data2, [dst, #-16] +#ifdef STPCPY + sub dstin, dst, #1 #endif - ret + ret -.Lmisaligned: - tbz src, #0, 1f - ldrb data1_w, [src], #1 - strb data1_w, [dst], #1 - cbnz data1_w, 1f -#if defined(STPCPY) - // Back up one so that dst points to the '\0' string terminator. - sub dst, dst, #1 +.Lpage_cross: + bic src, srcin, #15 + /* Start by loading two words at [srcin & ~15], then forcing the + bytes that precede srcin to 0xff. This means they never look + like termination bytes. */ + ldp data1, data2, [src] + lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */ + tst to_align, #7 + csetm tmp2, ne +#ifdef __AARCH64EB__ + lsl tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */ +#else + lsr tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */ #endif - ret -1: - tbz src, #1, 1f - ldrb data1_w, [src], #1 - strb data1_w, [dst], #1 - cbz data1_w, .Ldone - ldrb data2_w, [src], #1 - strb data2_w, [dst], #1 - cbnz data2_w, 1f -.Ldone: -#if defined(STPCPY) - // Back up one so that dst points to the '\0' string terminator. - sub dst, dst, #1 + orr data1, data1, tmp2 + orr data2a, data2, tmp2 + cmp to_align, #8 + csinv data1, data1, xzr, lt + csel data2, data2, data2a, lt + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + sub tmp3, data2, zeroones + orr tmp4, data2, #REP8_7f + bic has_nul1, tmp1, tmp2 + bics has_nul2, tmp3, tmp4 + ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */ + b.eq .Lpage_cross_ok + /* We now need to make data1 and data2 look like they've been + loaded directly from srcin. Do a rotate on the 128-bit value. */ + lsl tmp1, to_align, #3 /* Bytes->bits. */ + neg tmp2, to_align, lsl #3 +#ifdef __AARCH64EB__ + lsl data1a, data1, tmp1 + lsr tmp4, data2, tmp2 + lsl data2, data2, tmp1 + orr tmp4, tmp4, data1a + cmp to_align, #8 + csel data1, tmp4, data2, lt + rev tmp2, data1 + rev tmp4, data2 + sub tmp1, tmp2, zeroones + orr tmp2, tmp2, #REP8_7f + sub tmp3, tmp4, zeroones + orr tmp4, tmp4, #REP8_7f +#else + lsr data1a, data1, tmp1 + lsl tmp4, data2, tmp2 + lsr data2, data2, tmp1 + orr tmp4, tmp4, data1a + cmp to_align, #8 + csel data1, tmp4, data2, lt + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + sub tmp3, data2, zeroones + orr tmp4, data2, #REP8_7f #endif - ret -1: - tbz src, #2, 1f - ldr data1_w, [src], #4 - // Check for a zero. - sub has_nul1_w, data1_w, zeroones_w - bic has_nul1_w, has_nul1_w, data1_w - ands has_nul1_w, has_nul1_w, #0x80808080 - b.ne .Lnul_in_data1 - str data1_w, [dst], #4 -1: - tbz src, #3, .Lloop - ldr data1, [src], #8 - // Check for a zero. - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - bics has_nul1, tmp1, tmp2 - b.ne .Lnul_in_data1 - str data1, [dst], #8 - b .Lloop + bic has_nul1, tmp1, tmp2 + cbnz has_nul1, .Lfp_le8 + bic has_nul2, tmp3, tmp4 + b .Lfp_gt8 + #if defined(STPCPY) END(stpcpy) #elif defined(STRCPY) diff --git a/libc/arch-x86_64/string/sse2-memmove-slm.S b/libc/arch-x86_64/string/sse2-memmove-slm.S index 0dbffad2c..6a5afd610 100644 --- a/libc/arch-x86_64/string/sse2-memmove-slm.S +++ b/libc/arch-x86_64/string/sse2-memmove-slm.S @@ -91,9 +91,6 @@ name: \ .section .text.sse2,"ax",@progbits ENTRY (MEMMOVE) ENTRANCE -#ifdef USE_AS_BCOPY - xchg %rsi, %rdi -#endif mov %rdi, %rax /* Check whether we should copy backward or forward. */ diff --git a/libc/bionic/malloc_debug_check.cpp b/libc/bionic/malloc_debug_check.cpp index dee03fae1..da48059db 100644 --- a/libc/bionic/malloc_debug_check.cpp +++ b/libc/bionic/malloc_debug_check.cpp @@ -45,6 +45,7 @@ #include #include #include +#include #include "debug_mapinfo.h" #include "debug_stacktrace.h" @@ -55,6 +56,14 @@ #include "private/libc_logging.h" #include "private/ScopedPthreadMutexLocker.h" +extern unsigned int malloc_sig_enabled; +extern unsigned int min_allocation_report_limit; +extern unsigned int max_allocation_limit; +extern char* process_name; +static size_t total_count = 0; +static bool isDumped = false; +static bool sigHandled = false; + #define MAX_BACKTRACE_DEPTH 16 #define ALLOCATION_TAG 0x1ee7d00d #define BACKLOG_TAG 0xbabecafe @@ -63,6 +72,10 @@ #define FRONT_GUARD_LEN (1<<5) #define REAR_GUARD 0xbb #define REAR_GUARD_LEN (1<<5) +#define FRONT_GUARD_SS 0xab + +static void malloc_sigaction(int signum, siginfo_t * sg, void * cxt); +static struct sigaction default_sa; static void log_message(const char* format, ...) { va_list args; @@ -135,9 +148,14 @@ static inline void init_front_guard(hdr_t* hdr) { memset(hdr->front_guard, FRONT_GUARD, FRONT_GUARD_LEN); } +static inline void set_snapshot(hdr_t* hdr) { + memset(hdr->front_guard, FRONT_GUARD_SS, FRONT_GUARD_LEN); +} + static inline bool is_front_guard_valid(hdr_t* hdr) { for (size_t i = 0; i < FRONT_GUARD_LEN; i++) { - if (hdr->front_guard[i] != FRONT_GUARD) { + if (!((hdr->front_guard[i] == FRONT_GUARD) || + (hdr->front_guard[i] == FRONT_GUARD_SS))) { return false; } } @@ -171,6 +189,9 @@ static inline bool is_rear_guard_valid(hdr_t* hdr) { } static inline void add_locked(hdr_t* hdr, hdr_t** tail, hdr_t** head) { + if (hdr->tag == ALLOCATION_TAG) { + total_count += hdr->size; + } hdr->prev = NULL; hdr->next = *head; if (*head) @@ -181,6 +202,9 @@ static inline void add_locked(hdr_t* hdr, hdr_t** tail, hdr_t** head) { } static inline int del_locked(hdr_t* hdr, hdr_t** tail, hdr_t** head) { + if (hdr->tag == ALLOCATION_TAG) { + total_count -= hdr->size; + } if (hdr->prev) { hdr->prev->next = hdr->next; } else { @@ -194,6 +218,25 @@ static inline int del_locked(hdr_t* hdr, hdr_t** tail, hdr_t** head) { return 0; } +static void snapshot_report_leaked_nodes() { + log_message("%s: %s\n", __FILE__, __FUNCTION__); + hdr_t * iterator = head; + size_t total_size = 0; + do { + if (iterator->front_guard[0] == FRONT_GUARD && + iterator->size >= min_allocation_report_limit) { + log_message("obj %p, size %d", iterator, iterator->size); + total_size += iterator->size; + log_backtrace(iterator->bt, iterator->bt_depth); + log_message("------------------------------"); // as an end marker + // Marking the node as we do not want to print it again. + set_snapshot(iterator); + } + iterator = iterator->next; + } while (iterator); + log_message("Total Pending allocations after last snapshot: %d", total_size); +} + static inline void add(hdr_t* hdr, size_t size) { ScopedPthreadMutexLocker locker(&lock); hdr->tag = ALLOCATION_TAG; @@ -202,6 +245,21 @@ static inline void add(hdr_t* hdr, size_t size) { init_rear_guard(hdr); ++g_allocated_block_count; add_locked(hdr, &tail, &head); + if (total_count >= max_allocation_limit && !isDumped && malloc_sig_enabled) { + isDumped = true; + log_message("Maximum limit of the %s process (%d Bytes) size has reached."\ + "Maximum limit is set to:%d Bytes\n", process_name, + total_count, max_allocation_limit); + log_message("Start dumping allocations of the process %s", process_name); + log_message("+++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++ ***\n"); + + // Print allocations of the process + snapshot_report_leaked_nodes(); + + log_message("*** +++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++\n"); + log_message("Completed dumping allocations of the process %s", process_name); + + } } static inline int del(hdr_t* hdr) { @@ -233,7 +291,8 @@ static bool was_used_after_free(hdr_t* hdr) { static inline int check_guards(hdr_t* hdr, int* safe) { *safe = 1; if (!is_front_guard_valid(hdr)) { - if (hdr->front_guard[0] == FRONT_GUARD) { + if ((hdr->front_guard[0] == FRONT_GUARD) || + ((hdr->front_guard[0] == FRONT_GUARD_SS))) { log_message("+++ ALLOCATION %p SIZE %d HAS A CORRUPTED FRONT GUARD\n", user(hdr), hdr->size); } else { @@ -637,6 +696,8 @@ static void ReportMemoryLeaks() { pthread_key_t g_debug_calls_disabled; +#define DEBUG_SIGNAL SIGWINCH + extern "C" bool malloc_debug_initialize(HashTable* hash_table, const MallocDebug* malloc_dispatch) { g_hash_table = hash_table; g_malloc_dispatch = malloc_dispatch; @@ -656,6 +717,26 @@ extern "C" bool malloc_debug_initialize(HashTable* hash_table, const MallocDebug __libc_format_log(ANDROID_LOG_INFO, "libc", "not gathering backtrace information\n"); } +/* Initializes malloc debugging framework. + * See comments on MallocDebugInit in malloc_debug_common.h + */ + if (malloc_sig_enabled) { + struct sigaction sa; //local or static? + //struct sigaction sa_snapshot; //local or static? + sa.sa_handler = NULL; + sa.sa_sigaction = malloc_sigaction; + sigemptyset(&sa.sa_mask); + sigaddset(&sa.sa_mask, DEBUG_SIGNAL); + sa.sa_flags = SA_SIGINFO; + sa.sa_restorer = NULL; + if (sigaction(DEBUG_SIGNAL, &sa, &default_sa) < 0) { + log_message("Failed to register signal handler w/ errno %s", strerror(errno)); + malloc_sig_enabled = 0; + } else { + log_message("Registered signal handler"); + sigHandled = false; + } + } if (g_backtrace_enabled) { backtrace_startup(); } @@ -668,9 +749,67 @@ extern "C" void malloc_debug_finalize(int malloc_debug_level) { if (malloc_debug_level == 10) { ReportMemoryLeaks(); } + if (malloc_sig_enabled) { + log_message("Deregister %d signal handler", DEBUG_SIGNAL); + sigaction(DEBUG_SIGNAL, &default_sa, NULL); + malloc_sig_enabled = 0; + sigHandled = false; + } if (g_backtrace_enabled) { backtrace_shutdown(); } pthread_setspecific(g_debug_calls_disabled, NULL); } + +static void snapshot_nodes_locked() { + log_message("%s: %s\n", __FILE__, __FUNCTION__); + hdr_t * iterator = head; + do { + if (iterator->front_guard[0] == FRONT_GUARD) { + set_snapshot(iterator); + } + iterator = iterator->next; + } while (iterator); +} + +static void malloc_sigaction(int signum, siginfo_t * info, void * context) +{ + log_message("%s: %s\n", __FILE__, __FUNCTION__); + log_message("%s got %d signal from PID: %d (context:%x)\n", + __func__, signum, info->si_pid, context); + + if (signum != DEBUG_SIGNAL) { + log_message("RECEIVED %d instead of %d\n", signum, DEBUG_SIGNAL); + return; + } + + ScopedPthreadMutexLocker locker(&lock); + + log_message("Process under observation:%s", process_name); + log_message("Maximum process size limit:%d Bytes", max_allocation_limit); + log_message("Won't print allocation below %d Bytes", min_allocation_report_limit); + log_message("Total count: %d\n", total_count); + + if (!head) { + log_message("No allocations?"); + return; + } + // If sigHandled is false, meaning it's being handled first time + if (!sigHandled) { + sigHandled = true; + // Marking the nodes assuming that they should not be leaked nodes. + snapshot_nodes_locked(); + } else { + // We need to print new allocations now + log_message("Start dumping allocations of the process %s", process_name); + log_message("+++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++ ***\n"); + + // Print allocations of the process + snapshot_report_leaked_nodes(); + + log_message("*** +++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++\n"); + log_message("Completed dumping allocations of the process %s", process_name); + } + return; +} diff --git a/libc/bionic/malloc_debug_common.cpp b/libc/bionic/malloc_debug_common.cpp index 0b6a1421b..e0d439acf 100644 --- a/libc/bionic/malloc_debug_common.cpp +++ b/libc/bionic/malloc_debug_common.cpp @@ -295,6 +295,11 @@ extern "C" void* valloc(size_t bytes) { #include #include "private/libc_logging.h" +unsigned int malloc_sig_enabled = 0; +unsigned int max_allocation_limit; +unsigned int min_allocation_report_limit; +const char* process_name; + template static void InitMallocFunction(void* malloc_impl_handler, FunctionType* func, const char* prefix, const char* suffix) { char symbol[128]; @@ -396,6 +401,21 @@ static void malloc_init_impl() { } so_name = "libc_malloc_debug_qemu.so"; break; + case 40: + malloc_sig_enabled = 1; + char debug_proc_size[PROP_VALUE_MAX]; + if (__system_property_get("libc.debug.malloc.maxprocsize", debug_proc_size)) + max_allocation_limit = atoi(debug_proc_size); + else + max_allocation_limit = 30 * 1024 * 1024; // In Bytes [Default is 30 MB] + if (__system_property_get("libc.debug.malloc.minalloclim", debug_proc_size)) + min_allocation_report_limit = atoi(debug_proc_size); + else + min_allocation_report_limit = 10 * 1024; // In Bytes [Default is 10 KB] + process_name = getprogname(); + + so_name = "libc_malloc_debug_leak.so"; + break; default: error_log("%s: Debug level %d is unknown\n", getprogname(), g_malloc_debug_level); return; @@ -456,6 +476,9 @@ static void malloc_init_impl() { case 20: InitMalloc(malloc_impl_handle, &malloc_dispatch_table, "qemu_instrumented"); break; + case 40: + InitMalloc(malloc_impl_handle, &malloc_dispatch_table, "chk"); + break; default: break; } diff --git a/libc/bionic/mbstate.cpp b/libc/bionic/mbstate.cpp index cb327d839..35ae76a3c 100644 --- a/libc/bionic/mbstate.cpp +++ b/libc/bionic/mbstate.cpp @@ -47,11 +47,11 @@ __LIBC_HIDDEN__ uint8_t mbstate_get_byte(const mbstate_t* ps, int n) { __LIBC_HIDDEN__ size_t reset_and_return_illegal(int _errno, mbstate_t* ps) { errno = _errno; - *(reinterpret_cast(ps->__seq)) = 0; + ps->__seq32 = 0; return __MB_ERR_ILLEGAL_SEQUENCE; } __LIBC_HIDDEN__ size_t reset_and_return(int _return, mbstate_t* ps) { - *(reinterpret_cast(ps->__seq)) = 0; + ps->__seq32 = 0; return _return; } diff --git a/libc/bionic/md5.c b/libc/bionic/md5.c new file mode 100644 index 000000000..82bd576ad --- /dev/null +++ b/libc/bionic/md5.c @@ -0,0 +1,283 @@ +/* + * Copyright (c) 1995 - 2001 Kungliga Tekniska Högskolan + * (Royal Institute of Technology, Stockholm, Sweden). + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the Institute nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" + +__RCSID("$Heimdal: md5.c,v 1.15 2001/01/29 04:33:44 assar Exp $" + "$NetBSD: md5.c,v 1.1.1.4 2002/09/12 12:41:42 joda Exp $"); +#endif + +#include +#include "md5.h" +#include "hash.h" + +#define A m->counter[0] +#define B m->counter[1] +#define C m->counter[2] +#define D m->counter[3] +#define X data + +void +MD5_Init (struct md5 *m) +{ + m->sz[0] = 0; + m->sz[1] = 0; + D = 0x10325476; + C = 0x98badcfe; + B = 0xefcdab89; + A = 0x67452301; +} + +#define F(x,y,z) CRAYFIX((x & y) | (~x & z)) +#define G(x,y,z) CRAYFIX((x & z) | (y & ~z)) +#define H(x,y,z) (x ^ y ^ z) +#define I(x,y,z) CRAYFIX(y ^ (x | ~z)) + +#define DOIT(a,b,c,d,k,s,i,OP) \ +a = b + cshift(a + OP(b,c,d) + X[k] + (i), s) + +#define DO1(a,b,c,d,k,s,i) DOIT(a,b,c,d,k,s,i,F) +#define DO2(a,b,c,d,k,s,i) DOIT(a,b,c,d,k,s,i,G) +#define DO3(a,b,c,d,k,s,i) DOIT(a,b,c,d,k,s,i,H) +#define DO4(a,b,c,d,k,s,i) DOIT(a,b,c,d,k,s,i,I) + +static inline void +calc (struct md5 *m, u_int32_t *data) +{ + u_int32_t AA, BB, CC, DD; + + AA = A; + BB = B; + CC = C; + DD = D; + + /* Round 1 */ + + DO1(A,B,C,D,0,7,0xd76aa478); + DO1(D,A,B,C,1,12,0xe8c7b756); + DO1(C,D,A,B,2,17,0x242070db); + DO1(B,C,D,A,3,22,0xc1bdceee); + + DO1(A,B,C,D,4,7,0xf57c0faf); + DO1(D,A,B,C,5,12,0x4787c62a); + DO1(C,D,A,B,6,17,0xa8304613); + DO1(B,C,D,A,7,22,0xfd469501); + + DO1(A,B,C,D,8,7,0x698098d8); + DO1(D,A,B,C,9,12,0x8b44f7af); + DO1(C,D,A,B,10,17,0xffff5bb1); + DO1(B,C,D,A,11,22,0x895cd7be); + + DO1(A,B,C,D,12,7,0x6b901122); + DO1(D,A,B,C,13,12,0xfd987193); + DO1(C,D,A,B,14,17,0xa679438e); + DO1(B,C,D,A,15,22,0x49b40821); + + /* Round 2 */ + + DO2(A,B,C,D,1,5,0xf61e2562); + DO2(D,A,B,C,6,9,0xc040b340); + DO2(C,D,A,B,11,14,0x265e5a51); + DO2(B,C,D,A,0,20,0xe9b6c7aa); + + DO2(A,B,C,D,5,5,0xd62f105d); + DO2(D,A,B,C,10,9,0x2441453); + DO2(C,D,A,B,15,14,0xd8a1e681); + DO2(B,C,D,A,4,20,0xe7d3fbc8); + + DO2(A,B,C,D,9,5,0x21e1cde6); + DO2(D,A,B,C,14,9,0xc33707d6); + DO2(C,D,A,B,3,14,0xf4d50d87); + DO2(B,C,D,A,8,20,0x455a14ed); + + DO2(A,B,C,D,13,5,0xa9e3e905); + DO2(D,A,B,C,2,9,0xfcefa3f8); + DO2(C,D,A,B,7,14,0x676f02d9); + DO2(B,C,D,A,12,20,0x8d2a4c8a); + + /* Round 3 */ + + DO3(A,B,C,D,5,4,0xfffa3942); + DO3(D,A,B,C,8,11,0x8771f681); + DO3(C,D,A,B,11,16,0x6d9d6122); + DO3(B,C,D,A,14,23,0xfde5380c); + + DO3(A,B,C,D,1,4,0xa4beea44); + DO3(D,A,B,C,4,11,0x4bdecfa9); + DO3(C,D,A,B,7,16,0xf6bb4b60); + DO3(B,C,D,A,10,23,0xbebfbc70); + + DO3(A,B,C,D,13,4,0x289b7ec6); + DO3(D,A,B,C,0,11,0xeaa127fa); + DO3(C,D,A,B,3,16,0xd4ef3085); + DO3(B,C,D,A,6,23,0x4881d05); + + DO3(A,B,C,D,9,4,0xd9d4d039); + DO3(D,A,B,C,12,11,0xe6db99e5); + DO3(C,D,A,B,15,16,0x1fa27cf8); + DO3(B,C,D,A,2,23,0xc4ac5665); + + /* Round 4 */ + + DO4(A,B,C,D,0,6,0xf4292244); + DO4(D,A,B,C,7,10,0x432aff97); + DO4(C,D,A,B,14,15,0xab9423a7); + DO4(B,C,D,A,5,21,0xfc93a039); + + DO4(A,B,C,D,12,6,0x655b59c3); + DO4(D,A,B,C,3,10,0x8f0ccc92); + DO4(C,D,A,B,10,15,0xffeff47d); + DO4(B,C,D,A,1,21,0x85845dd1); + + DO4(A,B,C,D,8,6,0x6fa87e4f); + DO4(D,A,B,C,15,10,0xfe2ce6e0); + DO4(C,D,A,B,6,15,0xa3014314); + DO4(B,C,D,A,13,21,0x4e0811a1); + + DO4(A,B,C,D,4,6,0xf7537e82); + DO4(D,A,B,C,11,10,0xbd3af235); + DO4(C,D,A,B,2,15,0x2ad7d2bb); + DO4(B,C,D,A,9,21,0xeb86d391); + + A += AA; + B += BB; + C += CC; + D += DD; +} + +/* + * From `Performance analysis of MD5' by Joseph D. Touch + */ +#if !defined(__BYTE_ORDER) || !defined (__BIG_ENDIAN) +#error __BYTE_ORDER macros not defined +#endif + +#if __BYTE_ORDER == __BIG_ENDIAN +static inline u_int32_t +swap_u_int32_t (u_int32_t t) +{ + u_int32_t temp1, temp2; + + temp1 = cshift(t, 16); + temp2 = temp1 >> 8; + temp1 &= 0x00ff00ff; + temp2 &= 0x00ff00ff; + temp1 <<= 8; + return temp1 | temp2; +} +#endif + +struct x32{ + unsigned int a:32; + unsigned int b:32; +}; + +void +MD5_Update (struct md5 *m, const void *v, size_t len) +{ + const unsigned char *p = v; + size_t old_sz = m->sz[0]; + size_t offset; + + m->sz[0] += len * 8; + if (m->sz[0] < old_sz) + ++m->sz[1]; + offset = (old_sz / 8) % 64; + while(len > 0){ + size_t l = min(len, 64 - offset); + memcpy(m->save + offset, p, l); + offset += l; + p += l; + len -= l; + if(offset == 64){ +#if __BYTE_ORDER == __BIG_ENDIAN + int i; + u_int32_t current[16]; + struct x32 *u = (struct x32*)m->save; + for(i = 0; i < 8; i++){ + current[2*i+0] = swap_u_int32_t(u[i].a); + current[2*i+1] = swap_u_int32_t(u[i].b); + } + calc(m, current); +#else +<<<<<<< HEAD + calc(m, (u_int32_t*)m->save); +======= + calc(m, m->save32); +>>>>>>> b116fdf... Add md5 back to libc/bionic for targets that need it. +#endif + offset = 0; + } + } +} + +void +MD5_Final (void *res, struct md5 *m) +{ + unsigned char zeros[72]; + unsigned offset = (m->sz[0] / 8) % 64; + unsigned int dstart = (120 - offset - 1) % 64 + 1; + + *zeros = 0x80; + memset (zeros + 1, 0, sizeof(zeros) - 1); + zeros[dstart+0] = (m->sz[0] >> 0) & 0xff; + zeros[dstart+1] = (m->sz[0] >> 8) & 0xff; + zeros[dstart+2] = (m->sz[0] >> 16) & 0xff; + zeros[dstart+3] = (m->sz[0] >> 24) & 0xff; + zeros[dstart+4] = (m->sz[1] >> 0) & 0xff; + zeros[dstart+5] = (m->sz[1] >> 8) & 0xff; + zeros[dstart+6] = (m->sz[1] >> 16) & 0xff; + zeros[dstart+7] = (m->sz[1] >> 24) & 0xff; + MD5_Update (m, zeros, dstart + 8); + { + int i; + unsigned char *r = (unsigned char *)res; + + for (i = 0; i < 4; ++i) { + r[4*i] = m->counter[i] & 0xFF; + r[4*i+1] = (m->counter[i] >> 8) & 0xFF; + r[4*i+2] = (m->counter[i] >> 16) & 0xFF; + r[4*i+3] = (m->counter[i] >> 24) & 0xFF; + } + } +#if 0 + { + int i; + u_int32_t *r = (u_int32_t *)res; + + for (i = 0; i < 4; ++i) + r[i] = swap_u_int32_t (m->counter[i]); + } +#endif +} diff --git a/libc/bionic/md5.h b/libc/bionic/md5.h new file mode 100644 index 000000000..ddffb875b --- /dev/null +++ b/libc/bionic/md5.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 1995 - 2001 Kungliga Tekniska Högskolan + * (Royal Institute of Technology, Stockholm, Sweden). + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the Institute nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* $Heimdal: md5.h,v 1.8 2001/01/29 02:08:57 assar Exp $ + $NetBSD: md5.h,v 1.1.1.4 2002/09/12 12:41:42 joda Exp $ */ + +#include +#include + +struct md5 { + unsigned int sz[2]; + u_int32_t counter[4]; +<<<<<<< HEAD + unsigned char save[64]; +======= + union { + unsigned char save[64]; + u_int32_t save32[16]; + }; +>>>>>>> b116fdf... Add md5 back to libc/bionic for targets that need it. +}; + +typedef struct md5 MD5_CTX; + +void MD5_Init (struct md5 *m); +void MD5_Update (struct md5 *m, const void *p, size_t len); +void MD5_Final (void *res, struct md5 *m); /* u_int32_t res[4] */ diff --git a/libc/bionic/mmap.cpp b/libc/bionic/mmap.cpp index 8f25a8941..53e8b4615 100644 --- a/libc/bionic/mmap.cpp +++ b/libc/bionic/mmap.cpp @@ -36,6 +36,11 @@ extern "C" void* __mmap2(void*, size_t, int, int, int, size_t); #define MMAP2_SHIFT 12 // 2**12 == 4096 +#ifdef LEGACY_MMAP +#define TO_64(a) ((a) & 0x00000000ffffffff) +#else +#define TO_64(a) (a) +#endif static bool kernel_has_MADV_MERGEABLE = true; @@ -60,5 +65,5 @@ void* mmap64(void* addr, size_t size, int prot, int flags, int fd, off64_t offse } void* mmap(void* addr, size_t size, int prot, int flags, int fd, off_t offset) { - return mmap64(addr, size, prot, flags, fd, static_cast(offset)); + return mmap64(addr, size, prot, flags, fd, TO_64(static_cast(offset))); } diff --git a/libc/bionic/pty.cpp b/libc/bionic/pty.cpp index 995e00608..2c8618047 100644 --- a/libc/bionic/pty.cpp +++ b/libc/bionic/pty.cpp @@ -28,13 +28,15 @@ #include #include +#include #include #include #include #include #include +#include -int getpt(void) { +int getpt() { return posix_openpt(O_RDWR|O_NOCTTY); } @@ -47,7 +49,7 @@ int posix_openpt(int flags) { } char* ptsname(int fd) { - static char buf[64]; + static char buf[32]; return ptsname_r(fd, buf, sizeof(buf)) == 0 ? buf : NULL; } @@ -105,3 +107,83 @@ int unlockpt(int fd) { int unlock = 0; return ioctl(fd, TIOCSPTLCK, &unlock); } + +int openpty(int* master, int* slave, char* name, const termios* t, const winsize* ws) { + *master = getpt(); + if (*master == -1) { + return -1; + } + + if (grantpt(*master) == -1 || unlockpt(*master) == -1) { + close(*master); + return -1; + } + + char buf[32]; + if (name == NULL) { + name = buf; + } + if (ptsname_r(*master, name, sizeof(buf)) != 0) { + close(*master); + return -1; + } + + *slave = open(name, O_RDWR|O_NOCTTY); + if (*slave == -1) { + close(*master); + return -1; + } + + if (t != NULL) { + tcsetattr(*slave, TCSAFLUSH, t); + } + if (ws != NULL) { + ioctl(*slave, TIOCSWINSZ, ws); + } + + return 0; +} + +int forkpty(int* master, char* name, const termios* t, const winsize* ws) { + int slave; + if (openpty(master, &slave, name, t, ws) == -1) { + return -1; + } + + pid_t pid = fork(); + if (pid == -1) { + close(*master); + close(slave); + return -1; + } + + if (pid == 0) { + // Child. + close(*master); + if (login_tty(slave) == -1) { + _exit(1); + } + return 0; + } + + // Parent. + close(slave); + return pid; +} + +int login_tty(int fd) { + setsid(); + + if (ioctl(fd, TIOCSCTTY, NULL) == -1) { + return -1; + } + + dup2(fd, STDIN_FILENO); + dup2(fd, STDOUT_FILENO); + dup2(fd, STDERR_FILENO); + if (fd > STDERR_FILENO) { + close(fd); + } + + return 0; +} diff --git a/libc/bionic/wchar.cpp b/libc/bionic/wchar.cpp index e0879b9df..723639cb8 100644 --- a/libc/bionic/wchar.cpp +++ b/libc/bionic/wchar.cpp @@ -54,7 +54,7 @@ // int mbsinit(const mbstate_t* ps) { - return (ps == NULL || (*(reinterpret_cast(ps->__seq)) == 0)); + return (ps == NULL || ps->__seq32 == 0); } size_t mbrtowc(wchar_t* pwc, const char* s, size_t n, mbstate_t* ps) { diff --git a/libc/dns/net/getaddrinfo.c b/libc/dns/net/getaddrinfo.c index 2612d6a4d..ea1b5695f 100644 --- a/libc/dns/net/getaddrinfo.c +++ b/libc/dns/net/getaddrinfo.c @@ -1544,7 +1544,7 @@ _get_scope(const struct sockaddr *addr) /* RFC 4380, section 2.6 */ #define IN6_IS_ADDR_TEREDO(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == ntohl(0x20010000))) + (((a)->s6_addr32[0]) == ntohl(0x20010000)) /* RFC 3056, section 2. */ #define IN6_IS_ADDR_6TO4(a) \ diff --git a/libc/include/netinet/in6.h b/libc/include/netinet/in6.h index 7f3286ae9..f33e41ff5 100644 --- a/libc/include/netinet/in6.h +++ b/libc/include/netinet/in6.h @@ -31,28 +31,28 @@ #include #define IN6_IS_ADDR_UNSPECIFIED(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[4]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[8]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[12]) == 0)) + (((a)->s6_addr32[0] == 0) && \ + ((a)->s6_addr32[1] == 0) && \ + ((a)->s6_addr32[2] == 0) && \ + ((a)->s6_addr32[3] == 0)) #define IN6_IS_ADDR_LOOPBACK(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[4]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[8]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[12]) == ntohl(1))) + (((a)->s6_addr32[0] == 0) && \ + ((a)->s6_addr32[1] == 0) && \ + ((a)->s6_addr32[2] == 0) && \ + ((a)->s6_addr32[3] == ntohl(1))) #define IN6_IS_ADDR_V4COMPAT(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[4]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[8]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[12]) != 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[12]) != ntohl(1))) + (((a)->s6_addr32[0] == 0) && \ + ((a)->s6_addr32[1] == 0) && \ + ((a)->s6_addr32[2] == 0) && \ + ((a)->s6_addr32[3] != 0) && \ + ((a)->s6_addr32[3] != ntohl(1))) -#define IN6_IS_ADDR_V4MAPPED(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[4]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[8]) == ntohl(0x0000ffff))) +#define IN6_IS_ADDR_V4MAPPED(a) \ + (((a)->s6_addr32[0] == 0) && \ + ((a)->s6_addr32[1] == 0) && \ + ((a)->s6_addr32[2] == ntohl(0x0000ffff))) #define IN6_IS_ADDR_LINKLOCAL(a) \ (((a)->s6_addr[0] == 0xfe) && (((a)->s6_addr[1] & 0xc0) == 0x80)) @@ -65,7 +65,7 @@ (((a)->s6_addr[0] & 0xfe) == 0xfc) #define IN6_IS_ADDR_MULTICAST(a) \ - (((__const uint8_t *) (a))[0] == 0xff) + ((a)->s6_addr[0] == 0xff) #define IPV6_ADDR_SCOPE_NODELOCAL 0x01 diff --git a/libc/include/pty.h b/libc/include/pty.h new file mode 100644 index 000000000..bca113752 --- /dev/null +++ b/libc/include/pty.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _PTY_H +#define _PTY_H + +#include + +#include +#include + +__BEGIN_DECLS + +int openpty(int*, int*, char*, const struct termios*, const struct winsize*); +int forkpty(int*, char*, const struct termios*, const struct winsize*); + +__END_DECLS + +#endif /* _PTY_H */ diff --git a/libc/include/signal.h b/libc/include/signal.h index e23e65b80..8cb0678ec 100644 --- a/libc/include/signal.h +++ b/libc/include/signal.h @@ -34,7 +34,9 @@ #include /* For LONG_BIT */ #include /* For memset() */ #include +#if defined(__LP64__) #include +#endif #if defined(__LP64__) || defined(__mips__) /* For 64-bit (and mips), the kernel's struct sigaction doesn't match the POSIX one, diff --git a/libc/include/sys/_system_properties.h b/libc/include/sys/_system_properties.h index 0349e4c3d..0a842bd7c 100644 --- a/libc/include/sys/_system_properties.h +++ b/libc/include/sys/_system_properties.h @@ -83,7 +83,11 @@ struct prop_msg #define PROP_PATH_SYSTEM_DEFAULT "/system/default.prop" #define PROP_PATH_VENDOR_BUILD "/vendor/build.prop" #define PROP_PATH_LOCAL_OVERRIDE "/data/local.prop" +#ifdef OVERRIDE_PROP_PATH_FACTORY +#define PROP_PATH_FACTORY OVERRIDE_PROP_PATH_FACTORY +#else #define PROP_PATH_FACTORY "/factory/factory.prop" +#endif /* ** Map the property area from the specified filename. This diff --git a/libc/include/uchar.h b/libc/include/uchar.h index e1fcb5c65..0ec9d2eb5 100644 --- a/libc/include/uchar.h +++ b/libc/include/uchar.h @@ -34,6 +34,11 @@ __BEGIN_DECLS +#if defined(__GNUC__) && __GNUC__ >= 5 && !defined(__cplusplus) +typedef __CHAR16_TYPE__ char16_t; +typedef __CHAR32_TYPE__ char32_t; +#endif + #define __STD_UTF_16__ 1 #define __STD_UTF_32__ 1 diff --git a/libc/include/utmp.h b/libc/include/utmp.h index d76422781..ebf23726a 100644 --- a/libc/include/utmp.h +++ b/libc/include/utmp.h @@ -91,6 +91,8 @@ int utmpname(const char*); void setutent(); struct utmp* getutent(); +int login_tty(int); + __END_DECLS #endif /* _UTMP_H_ */ diff --git a/libc/include/wchar.h b/libc/include/wchar.h index e0e5c82fe..dc328889f 100644 --- a/libc/include/wchar.h +++ b/libc/include/wchar.h @@ -41,7 +41,10 @@ __BEGIN_DECLS typedef __WINT_TYPE__ wint_t; typedef struct { - uint8_t __seq[4]; + union { + uint8_t __seq[4]; + uint32_t __seq32; + }; #ifdef __LP64__ char __reserved[4]; #endif diff --git a/libc/stdio/findfp.c b/libc/stdio/findfp.c index cfbb66bb8..7a537c8fe 100644 --- a/libc/stdio/findfp.c +++ b/libc/stdio/findfp.c @@ -53,8 +53,8 @@ int __sdidinit; #define NDYNAMIC 10 /* add ten more whenever necessary */ #define std(flags, file) \ - {0,0,0,flags,file,{0},0,__sF+file,__sclose,__sread,__sseek,__swrite, \ - {(unsigned char *)(__sFext+file), 0},NULL,0,{0},{0},{0},0,0} + {0,0,0,flags,file,{0,0},0,__sF+file,__sclose,__sread,__sseek,__swrite, \ + {(unsigned char *)(__sFext+file), 0},NULL,0,{0},{0},{0,0},0,0} /* the usual - (stdin + stdout + stderr) */ static FILE usual[FOPEN_MAX - 3]; diff --git a/libm/Android.mk b/libm/Android.mk index 994caa0c6..7932aea2a 100644 --- a/libm/Android.mk +++ b/libm/Android.mk @@ -16,6 +16,8 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/bsdsrc/b_exp.c \ upstream-freebsd/lib/msun/bsdsrc/b_log.c \ upstream-freebsd/lib/msun/bsdsrc/b_tgamma.c \ + upstream-freebsd/lib/msun/src/catrig.c \ + upstream-freebsd/lib/msun/src/catrigf.c \ upstream-freebsd/lib/msun/src/e_acos.c \ upstream-freebsd/lib/msun/src/e_acosf.c \ upstream-freebsd/lib/msun/src/e_acosh.c \ @@ -64,8 +66,6 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/e_scalbf.c \ upstream-freebsd/lib/msun/src/e_sinh.c \ upstream-freebsd/lib/msun/src/e_sinhf.c \ - upstream-freebsd/lib/msun/src/e_sqrt.c \ - upstream-freebsd/lib/msun/src/e_sqrtf.c \ upstream-freebsd/lib/msun/src/imprecise.c \ upstream-freebsd/lib/msun/src/k_cos.c \ upstream-freebsd/lib/msun/src/k_cosf.c \ @@ -82,30 +82,33 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_atanf.c \ upstream-freebsd/lib/msun/src/s_carg.c \ upstream-freebsd/lib/msun/src/s_cargf.c \ + upstream-freebsd/lib/msun/src/s_cargl.c \ upstream-freebsd/lib/msun/src/s_cbrt.c \ upstream-freebsd/lib/msun/src/s_cbrtf.c \ upstream-freebsd/lib/msun/src/s_ccosh.c \ upstream-freebsd/lib/msun/src/s_ccoshf.c \ - upstream-freebsd/lib/msun/src/s_ceil.c \ - upstream-freebsd/lib/msun/src/s_ceilf.c \ upstream-freebsd/lib/msun/src/s_cexp.c \ upstream-freebsd/lib/msun/src/s_cexpf.c \ upstream-freebsd/lib/msun/src/s_cimag.c \ upstream-freebsd/lib/msun/src/s_cimagf.c \ + upstream-freebsd/lib/msun/src/s_cimagl.c \ upstream-freebsd/lib/msun/src/s_conj.c \ upstream-freebsd/lib/msun/src/s_conjf.c \ + upstream-freebsd/lib/msun/src/s_conjl.c \ upstream-freebsd/lib/msun/src/s_copysign.c \ upstream-freebsd/lib/msun/src/s_copysignf.c \ - upstream-freebsd/lib/msun/src/s_cos.c \ upstream-freebsd/lib/msun/src/s_cosf.c \ upstream-freebsd/lib/msun/src/s_cproj.c \ upstream-freebsd/lib/msun/src/s_cprojf.c \ + upstream-freebsd/lib/msun/src/s_cprojl.c \ upstream-freebsd/lib/msun/src/s_creal.c \ upstream-freebsd/lib/msun/src/s_crealf.c \ + upstream-freebsd/lib/msun/src/s_creall.c \ upstream-freebsd/lib/msun/src/s_csinh.c \ upstream-freebsd/lib/msun/src/s_csinhf.c \ upstream-freebsd/lib/msun/src/s_csqrt.c \ upstream-freebsd/lib/msun/src/s_csqrtf.c \ + upstream-freebsd/lib/msun/src/s_csqrtl.c \ upstream-freebsd/lib/msun/src/s_ctanh.c \ upstream-freebsd/lib/msun/src/s_ctanhf.c \ upstream-freebsd/lib/msun/src/s_erf.c \ @@ -119,10 +122,6 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_fdim.c \ upstream-freebsd/lib/msun/src/s_finite.c \ upstream-freebsd/lib/msun/src/s_finitef.c \ - upstream-freebsd/lib/msun/src/s_floor.c \ - upstream-freebsd/lib/msun/src/s_floorf.c \ - upstream-freebsd/lib/msun/src/s_fma.c \ - upstream-freebsd/lib/msun/src/s_fmaf.c \ upstream-freebsd/lib/msun/src/s_fmax.c \ upstream-freebsd/lib/msun/src/s_fmaxf.c \ upstream-freebsd/lib/msun/src/s_fmin.c \ @@ -131,16 +130,12 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_frexpf.c \ upstream-freebsd/lib/msun/src/s_ilogb.c \ upstream-freebsd/lib/msun/src/s_ilogbf.c \ - upstream-freebsd/lib/msun/src/s_llrint.c \ - upstream-freebsd/lib/msun/src/s_llrintf.c \ upstream-freebsd/lib/msun/src/s_llround.c \ upstream-freebsd/lib/msun/src/s_llroundf.c \ upstream-freebsd/lib/msun/src/s_log1p.c \ upstream-freebsd/lib/msun/src/s_log1pf.c \ upstream-freebsd/lib/msun/src/s_logb.c \ upstream-freebsd/lib/msun/src/s_logbf.c \ - upstream-freebsd/lib/msun/src/s_lrint.c \ - upstream-freebsd/lib/msun/src/s_lrintf.c \ upstream-freebsd/lib/msun/src/s_lround.c \ upstream-freebsd/lib/msun/src/s_lroundf.c \ upstream-freebsd/lib/msun/src/s_modf.c \ @@ -151,8 +146,6 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_nextafterf.c \ upstream-freebsd/lib/msun/src/s_remquo.c \ upstream-freebsd/lib/msun/src/s_remquof.c \ - upstream-freebsd/lib/msun/src/s_rint.c \ - upstream-freebsd/lib/msun/src/s_rintf.c \ upstream-freebsd/lib/msun/src/s_round.c \ upstream-freebsd/lib/msun/src/s_roundf.c \ upstream-freebsd/lib/msun/src/s_scalbln.c \ @@ -161,25 +154,23 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_signgam.c \ upstream-freebsd/lib/msun/src/s_significand.c \ upstream-freebsd/lib/msun/src/s_significandf.c \ - upstream-freebsd/lib/msun/src/s_sin.c \ upstream-freebsd/lib/msun/src/s_sinf.c \ upstream-freebsd/lib/msun/src/s_tan.c \ upstream-freebsd/lib/msun/src/s_tanf.c \ upstream-freebsd/lib/msun/src/s_tanh.c \ upstream-freebsd/lib/msun/src/s_tanhf.c \ upstream-freebsd/lib/msun/src/s_tgammaf.c \ - upstream-freebsd/lib/msun/src/s_trunc.c \ - upstream-freebsd/lib/msun/src/s_truncf.c \ upstream-freebsd/lib/msun/src/w_cabs.c \ upstream-freebsd/lib/msun/src/w_cabsf.c \ + upstream-freebsd/lib/msun/src/w_cabsl.c \ upstream-freebsd/lib/msun/src/w_drem.c \ - upstream-freebsd/lib/msun/src/w_dremf.c \ + upstream-freebsd/lib/msun/src/w_dremf.c libm_common_src_files += \ fake_long_double.c \ signbit.c \ -libm_ld_src_files = \ +libm_ld128_src_files = \ upstream-freebsd/lib/msun/src/e_acosl.c \ upstream-freebsd/lib/msun/src/e_acoshl.c \ upstream-freebsd/lib/msun/src/e_asinl.c \ @@ -187,6 +178,7 @@ libm_ld_src_files = \ upstream-freebsd/lib/msun/src/e_atanhl.c \ upstream-freebsd/lib/msun/src/e_fmodl.c \ upstream-freebsd/lib/msun/src/e_hypotl.c \ + upstream-freebsd/lib/msun/src/e_lgammal.c \ upstream-freebsd/lib/msun/src/e_remainderl.c \ upstream-freebsd/lib/msun/src/e_sqrtl.c \ upstream-freebsd/lib/msun/src/s_asinhl.c \ @@ -194,6 +186,7 @@ libm_ld_src_files = \ upstream-freebsd/lib/msun/src/s_cbrtl.c \ upstream-freebsd/lib/msun/src/s_ceill.c \ upstream-freebsd/lib/msun/src/s_copysignl.c \ + upstream-freebsd/lib/msun/src/e_coshl.c \ upstream-freebsd/lib/msun/src/s_cosl.c \ upstream-freebsd/lib/msun/src/s_fabsl.c \ upstream-freebsd/lib/msun/src/s_floorl.c \ @@ -215,25 +208,164 @@ libm_ld_src_files = \ upstream-freebsd/lib/msun/src/s_rintl.c \ upstream-freebsd/lib/msun/src/s_roundl.c \ upstream-freebsd/lib/msun/src/s_scalbnl.c \ + upstream-freebsd/lib/msun/src/e_sinhl.c \ upstream-freebsd/lib/msun/src/s_sinl.c \ + upstream-freebsd/lib/msun/src/s_tanhl.c \ upstream-freebsd/lib/msun/src/s_tanl.c \ upstream-freebsd/lib/msun/src/s_truncl.c \ -libm_ld_src_files += \ +libm_ld128_src_files += \ upstream-freebsd/lib/msun/ld128/invtrig.c \ + upstream-freebsd/lib/msun/ld128/e_lgammal_r.c \ upstream-freebsd/lib/msun/ld128/k_cosl.c \ upstream-freebsd/lib/msun/ld128/k_sinl.c \ upstream-freebsd/lib/msun/ld128/k_tanl.c \ + upstream-freebsd/lib/msun/ld128/s_erfl.c \ upstream-freebsd/lib/msun/ld128/s_exp2l.c \ upstream-freebsd/lib/msun/ld128/s_expl.c \ upstream-freebsd/lib/msun/ld128/s_logl.c \ upstream-freebsd/lib/msun/ld128/s_nanl.c \ +# Arch specific optimizations. + +# ----------------------------------------------------------------------------- +# arm +# ----------------------------------------------------------------------------- +libm_arm_arch_files += \ + upstream-freebsd/lib/msun/src/s_ceil.c \ + upstream-freebsd/lib/msun/src/s_ceilf.c \ + upstream-freebsd/lib/msun/src/s_fma.c \ + upstream-freebsd/lib/msun/src/s_fmaf.c \ + upstream-freebsd/lib/msun/src/s_floorf.c \ + upstream-freebsd/lib/msun/src/s_llrint.c \ + upstream-freebsd/lib/msun/src/s_llrintf.c \ + upstream-freebsd/lib/msun/src/s_lrint.c \ + upstream-freebsd/lib/msun/src/s_lrintf.c \ + upstream-freebsd/lib/msun/src/s_rint.c \ + upstream-freebsd/lib/msun/src/s_rintf.c \ + upstream-freebsd/lib/msun/src/s_trunc.c \ + upstream-freebsd/lib/msun/src/s_truncf.c \ + +# s_floor.S requires neon instructions. +ifdef TARGET_2ND_ARCH +arch_variant := $(TARGET_2ND_ARCH_VARIANT) +else +arch_variant := $(TARGET_ARCH_VARIANT) +endif + +# Use the C version on armv7-a since it doesn't support neon instructions. +ifeq ($(arch_variant),armv7-a) +libm_arm_arch_files += \ + upstream-freebsd/lib/msun/src/s_cos.c \ + upstream-freebsd/lib/msun/src/s_sin.c \ + upstream-freebsd/lib/msun/src/e_sqrt.c \ + upstream-freebsd/lib/msun/src/e_sqrtf.c \ + upstream-freebsd/lib/msun/src/s_floor.c \ + +else +libm_arm_arch_files += \ + arm/s_cos.S \ + arm/s_sin.S \ + arm/e_sqrt.S \ + arm/e_sqrtf.S \ + arm/s_floor.S \ + +libm_arm_arch_cflags += -DQCOM_NEON_OPTIMIZATION -fno-if-conversion +endif + +# ----------------------------------------------------------------------------- +# arm64 +# ----------------------------------------------------------------------------- +libm_arm64_arch_files += \ + upstream-freebsd/lib/msun/src/s_cos.c \ + upstream-freebsd/lib/msun/src/s_sin.c \ + arm64/ceil.S \ + arm64/fma.S \ + arm64/floor.S \ + arm64/lrint.S \ + arm64/rint.S \ + arm64/sqrt.S \ + arm64/trunc.S \ + +libm_arm64_arch_cflags += -DQCOM_NEON_OPTIMIZATION + +# ----------------------------------------------------------------------------- +# mips +# ----------------------------------------------------------------------------- +libm_mips_arch_files += \ + mips/fenv.c \ + upstream-freebsd/lib/msun/src/e_sqrt.c \ + upstream-freebsd/lib/msun/src/e_sqrtf.c \ + upstream-freebsd/lib/msun/src/s_ceil.c \ + upstream-freebsd/lib/msun/src/s_ceilf.c \ + upstream-freebsd/lib/msun/src/s_cos.c \ + upstream-freebsd/lib/msun/src/s_fma.c \ + upstream-freebsd/lib/msun/src/s_fmaf.c \ + upstream-freebsd/lib/msun/src/s_floor.c \ + upstream-freebsd/lib/msun/src/s_floorf.c \ + upstream-freebsd/lib/msun/src/s_llrint.c \ + upstream-freebsd/lib/msun/src/s_llrintf.c \ + upstream-freebsd/lib/msun/src/s_lrint.c \ + upstream-freebsd/lib/msun/src/s_lrintf.c \ + upstream-freebsd/lib/msun/src/s_rint.c \ + upstream-freebsd/lib/msun/src/s_rintf.c \ + upstream-freebsd/lib/msun/src/s_sin.c \ + upstream-freebsd/lib/msun/src/s_trunc.c \ + upstream-freebsd/lib/msun/src/s_truncf.c \ + + +# ----------------------------------------------------------------------------- +# x86 +# ----------------------------------------------------------------------------- +libm_x86_arch_files += \ + upstream-freebsd/lib/msun/src/e_sqrt.c \ + upstream-freebsd/lib/msun/src/e_sqrtf.c \ + upstream-freebsd/lib/msun/src/s_ceil.c \ + upstream-freebsd/lib/msun/src/s_ceilf.c \ + upstream-freebsd/lib/msun/src/s_cos.c \ + upstream-freebsd/lib/msun/src/s_fma.c \ + upstream-freebsd/lib/msun/src/s_fmaf.c \ + upstream-freebsd/lib/msun/src/s_floor.c \ + upstream-freebsd/lib/msun/src/s_floorf.c \ + upstream-freebsd/lib/msun/src/s_llrint.c \ + upstream-freebsd/lib/msun/src/s_llrintf.c \ + upstream-freebsd/lib/msun/src/s_lrint.c \ + upstream-freebsd/lib/msun/src/s_lrintf.c \ + upstream-freebsd/lib/msun/src/s_rint.c \ + upstream-freebsd/lib/msun/src/s_rintf.c \ + upstream-freebsd/lib/msun/src/s_sin.c \ + upstream-freebsd/lib/msun/src/s_trunc.c \ + upstream-freebsd/lib/msun/src/s_truncf.c \ + +# ----------------------------------------------------------------------------- +# x86_64 +# ----------------------------------------------------------------------------- +libm_x86_64_arch_files += \ + upstream-freebsd/lib/msun/src/e_sqrt.c \ + upstream-freebsd/lib/msun/src/e_sqrtf.c \ + upstream-freebsd/lib/msun/src/s_ceil.c \ + upstream-freebsd/lib/msun/src/s_ceilf.c \ + upstream-freebsd/lib/msun/src/s_cos.c \ + upstream-freebsd/lib/msun/src/s_fma.c \ + upstream-freebsd/lib/msun/src/s_fmaf.c \ + upstream-freebsd/lib/msun/src/s_floor.c \ + upstream-freebsd/lib/msun/src/s_floorf.c \ + upstream-freebsd/lib/msun/src/s_llrint.c \ + upstream-freebsd/lib/msun/src/s_llrintf.c \ + upstream-freebsd/lib/msun/src/s_lrint.c \ + upstream-freebsd/lib/msun/src/s_lrintf.c \ + upstream-freebsd/lib/msun/src/s_rint.c \ + upstream-freebsd/lib/msun/src/s_rintf.c \ + upstream-freebsd/lib/msun/src/s_sin.c \ + upstream-freebsd/lib/msun/src/s_trunc.c \ + upstream-freebsd/lib/msun/src/s_truncf.c \ + # TODO: re-enable i387/e_sqrtf.S for x86, and maybe others. libm_common_cflags := \ -DFLT_EVAL_METHOD=0 \ -std=c99 \ + -O3 \ -include $(LOCAL_PATH)/freebsd-compat.h \ -Wno-missing-braces \ -Wno-parentheses \ @@ -246,10 +378,9 @@ libm_common_cflags := \ # self recursions for lrint, lrintf, and lrintl. # BUG: 14225968 libm_common_cflags += -fno-builtin-rint -fno-builtin-rintf -fno-builtin-rintl +libm_common_includes += $(LOCAL_PATH)/upstream-freebsd/lib/msun/src/ -libm_common_includes := $(LOCAL_PATH)/upstream-freebsd/lib/msun/src/ - -libm_ld_includes := $(LOCAL_PATH)/upstream-freebsd/lib/msun/ld128/ +libm_ld_includes += $(LOCAL_PATH)/upstream-freebsd/lib/msun/ld128/ # # libm.a for target. @@ -263,23 +394,29 @@ LOCAL_C_INCLUDES += $(libm_common_includes) LOCAL_SRC_FILES := $(libm_common_src_files) LOCAL_SYSTEM_SHARED_LIBRARIES := libc +LOCAL_ASFLAGS := \ + -Ibionic/libc \ + # arch-specific settings -LOCAL_C_INCLUDES_arm := $(LOCAL_PATH)/arm -LOCAL_SRC_FILES_arm := arm/fenv.c +LOCAL_CFLAGS_arm := $(libm_arm_arch_cflags) +LOCAL_C_INCLUDES_arm := $(LOCAL_PATH)/arm $(libm_arm_arch_includes) +LOCAL_SRC_FILES_arm := arm/fenv.c $(libm_arm_arch_files) -LOCAL_C_INCLUDES_arm64 := $(libm_ld_includes) -LOCAL_SRC_FILES_arm64 := arm64/fenv.c $(libm_ld_src_files) +LOCAL_CFLAGS_arm64 := $(libm_arm64_arch_cflags) +LOCAL_C_INCLUDES_arm64 := $(libm_ld_includes) $(libm_arm64_arch_includes) +LOCAL_SRC_FILES_arm64 := arm64/fenv.c $(libm_ld128_src_files) $(libm_arm64_arch_files) LOCAL_C_INCLUDES_x86 := $(LOCAL_PATH)/i387 -LOCAL_SRC_FILES_x86 := i387/fenv.c +LOCAL_SRC_FILES_x86 := i387/fenv.c $(libm_x86_arch_files) LOCAL_C_INCLUDES_x86_64 := $(libm_ld_includes) -LOCAL_SRC_FILES_x86_64 := amd64/fenv.c $(libm_ld_src_files) +LOCAL_SRC_FILES_x86_64 := amd64/fenv.c $(libm_ld128_src_files) $(libm_x86_64_arch_files) -LOCAL_SRC_FILES_mips := mips/fenv.c +LOCAL_SRC_FILES_mips += mips/fenv.c $(libm_mips_arch_files) LOCAL_C_INCLUDES_mips64 := $(libm_ld_includes) -LOCAL_SRC_FILES_mips64 := mips/fenv.c $(libm_ld_src_files) +LOCAL_SRC_FILES_mips64 := mips/fenv.c $(libm_ld128_src_files) +LOCAL_SRC_FILES_mips64 += $(libm_mips_arch_files) include $(BUILD_STATIC_LIBRARY) diff --git a/libm/arm/e_pow.S b/libm/arm/e_pow.S new file mode 100644 index 000000000..a730fa657 --- /dev/null +++ b/libm/arm/e_pow.S @@ -0,0 +1,459 @@ +@ Copyright (c) 2009-2013 The Linux Foundation. All rights reserved. +@ +@ Redistribution and use in source and binary forms, with or without +@ modification, are permitted provided that the following conditions are met: +@ * Redistributions of source code must retain the above copyright +@ notice, this list of conditions and the following disclaimer. +@ * Redistributions in binary form must reproduce the above copyright +@ notice, this list of conditions and the following disclaimer in the +@ documentation and/or other materials provided with the distribution. +@ * Neither the name of The Linux Foundation nor the names of its contributors may +@ be used to endorse or promote products derived from this software +@ without specific prior written permission. +@ +@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +@ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +@ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +@ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +@ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +@ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +@ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +@ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +@ POSSIBILITY OF SUCH DAMAGE. + + +#include +#include + +@ Values which exist the program lifetime: +#define HIGH_WORD_MASK d31 +#define EXPONENT_MASK d30 +#define int_1 d29 +#define double_1 d28 +@ sign and 2^int_n fixup: +#define maxrange r12 +#define expadjustment d7 +#define literals r10 +@ Values which exist within both polynomial implementations: +#define int_n d2 +#define int_n_low s4 +#define int_n_high s5 +#define double_n d3 +#define k1 d27 +#define k2 d26 +#define k3 d25 +#define k4 d24 +@ Values which cross the boundaries between polynomial implementations: +#define ss d16 +#define ss2 d17 +#define ss4 d18 +#define Result d0 +#define Return_hw r1 +#define Return_lw r0 +#define ylg2x d0 +@ Intermediate values only needed sometimes: +@ initial (sorted in approximate order of availability for overwriting): +#define x_hw r1 +#define x_lw r0 +#define y_hw r3 +#define y_lw r2 +#define x d0 +#define bp d4 +#define y d1 +@ log series: +#define u d19 +#define v d20 +#define lg2coeff d21 +#define bpa d5 +#define bpb d3 +#define lg2const d6 +#define xmantissa r8 +#define twoto1o5 r4 +#define twoto3o5 r5 +#define ix r6 +#define iEXP_MASK r7 +@ exp input setup: +#define twoto1o8mask d3 +#define twoto1o4mask d4 +#define twoto1o2mask d1 +#define ylg2x_round_offset d16 +#define ylg2x_temp d17 +#define yn_temp d18 +#define yn_round_offset d19 +#define ln2 d5 +@ Careful, overwriting HIGH_WORD_MASK, reset it if you need it again ... +#define rounded_exponent d31 +@ exp series: +#define k5 d23 +#define k6 d22 +#define k7 d21 +#define k8 d20 +#define ss3 d19 +@ overwrite double_1 (we're done with it by now) +#define k0 d28 +#define twoto1o4 d6 + +@instructions that gas doesn't like to encode correctly: +#define vmov_f64 fconstd +#define vmov_f32 fconsts +#define vmovne_f64 fconstdne + +#define KRAIT_NO_AAPCS_VFP_MODE + +ENTRY(pow) +#if defined(KRAIT_NO_AAPCS_VFP_MODE) + @ ARM ABI has inputs coming in via r registers, lets move to a d register + vmov x, x_lw, x_hw +#endif + push {r4, r5, r6, r7, r8, r9, r10, lr} + + movw maxrange, #0x0000 + movt maxrange, #0x4010 + + @ pre-staged bp values + vldr bpa, .LbpA + vldr bpb, .LbpB + @ load two fifths into constant term in case we need it due to offsets + vldr lg2const, .Ltwofifths + + @ bp is initially 1.0, may adjust later based on x value + vmov_f64 bp, #0x70 + + @ extract the mantissa from x for scaled value comparisons + lsl xmantissa, x_hw, #12 + + @ twoto1o5 = 2^(1/5) (input bracketing) + movw twoto1o5, #0x186c + movt twoto1o5, #0x2611 + @ twoto3o5 = 2^(3/5) (input bracketing) + movw twoto3o5, #0x003b + movt twoto3o5, #0x8406 + + @ finish extracting xmantissa + orr xmantissa, xmantissa, x_lw, lsr #20 + + @ begin preparing a mask for normalization + vmov.i64 HIGH_WORD_MASK, #0xffffffff00000000 + + @ double_1 = (double) 1.0 + vmov_f64 double_1, #0x70 + +#if defined(KRAIT_NO_AAPCS_VFP_MODE) + @ move y from r registers to a d register + vmov y, y_lw, y_hw +#endif + + cmp xmantissa, twoto1o5 + + vshl.i64 EXPONENT_MASK, HIGH_WORD_MASK, #20 + vshr.u64 int_1, HIGH_WORD_MASK, #63 + + adr literals, .LliteralTable + + bhi .Lxgt2to1over5 + @ zero out lg2 constant term if don't offset our input + vsub.f64 lg2const, lg2const, lg2const + b .Lxle2to1over5 + +.Lxgt2to1over5: + @ if normalized x > 2^(1/5), bp = 1 + (2^(2/5)-1) = 2^(2/5) + vadd.f64 bp, bp, bpa + +.Lxle2to1over5: + @ will need ln2 for various things + vldr ln2, .Lln2 + + cmp xmantissa, twoto3o5 +@@@@ X Value Normalization @@@@ + + @ ss = abs(x) 2^(-1024) + vbic.i64 ss, x, EXPONENT_MASK + + @ N = (floor(log2(x)) + 0x3ff) * 2^52 + vand.i64 int_n, x, EXPONENT_MASK + + bls .Lxle2to3over5 + @ if normalized x > 2^(3/5), bp = 2^(2/5) + (2^(4/5) - 2^(2/5) = 2^(4/5) + vadd.f64 bp, bp, bpb + vadd.f64 lg2const, lg2const, lg2const + +.Lxle2to3over5: + + cmp x_hw, maxrange + cmpls y_hw, maxrange + movt maxrange, #0x3f00 + cmpls maxrange, x_hw + + @ load log2 polynomial series constants + vldm literals!, {k4, k3, k2, k1} + + @ s = abs(x) 2^(-floor(log2(x))) (normalize abs(x) to around 1) + vorr.i64 ss, ss, double_1 + +@@@@ 3/2 (Log(bp(1+s)/(1-s))) input computation (s = (x-bp)/(x+bp)) @@@@ + + vsub.f64 u, ss, bp + vadd.f64 v, ss, bp + + bhi .LuseFullImpl + + @ s = (x-1)/(x+1) + vdiv.f64 ss, u, v + + @ load 2/(3log2) into lg2coeff + vldr lg2coeff, .Ltwooverthreeln2 + + @ N = floor(log2(x)) * 2^52 + vsub.i64 int_n, int_n, double_1 + +@@@@ 3/2 (Log(bp(1+s)/(1-s))) polynomial series @@@@ + + @ ss2 = ((x-dp)/(x+dp))^2 + vmul.f64 ss2, ss, ss + @ ylg2x = 3.0 + vmov_f64 ylg2x, #8 + vmul.f64 ss4, ss2, ss2 + + @ todo: useful later for two-way clamp + vmul.f64 lg2coeff, lg2coeff, y + + @ N = floor(log2(x)) + vshr.s64 int_n, int_n, #52 + + @ k3 = ss^2 * L4 + L3 + vmla.f64 k3, ss2, k4 + + @ k1 = ss^2 * L2 + L1 + vmla.f64 k1, ss2, k2 + + @ scale ss by 2/(3 ln 2) + vmul.f64 lg2coeff, ss, lg2coeff + + @ ylg2x = 3.0 + s^2 + vadd.f64 ylg2x, ylg2x, ss2 + + vcvt.f64.s32 double_n, int_n_low + + @ k1 = s^4 (s^2 L4 + L3) + s^2 L2 + L1 + vmla.f64 k1, ss4, k3 + + @ add in constant term + vadd.f64 double_n, lg2const + + @ ylg2x = 3.0 + s^2 + s^4 (s^4 (s^2 L4 + L3) + s^2 L2 + L1) + vmla.f64 ylg2x, ss4, k1 + + @ ylg2x = y 2 s / (3 ln(2)) (3.0 + s^2 + s^4 (s^4(s^2 L4 + L3) + s^2 L2 + L1) + vmul.f64 ylg2x, lg2coeff, ylg2x + +@@@@ Compute input to Exp(s) (s = y(n + log2(x)) - (floor(8 yn + 1)/8 + floor(8 ylog2(x) + 1)/8) @@@@@ + + @ mask to extract bit 1 (2^-2 from our fixed-point representation) + vshl.u64 twoto1o4mask, int_1, #1 + + @ double_n = y * n + vmul.f64 double_n, double_n, y + + @ Load 2^(1/4) for later computations + vldr twoto1o4, .Ltwoto1o4 + + @ either add or subtract one based on the sign of double_n and ylg2x + vshr.s64 ylg2x_round_offset, ylg2x, #62 + vshr.s64 yn_round_offset, double_n, #62 + + @ move unmodified y*lg2x into temp space + vmov ylg2x_temp, ylg2x + @ compute floor(8 y * n + 1)/8 + @ and floor(8 y (log2(x)) + 1)/8 + vcvt.s32.f64 ylg2x, ylg2x, #3 + @ move unmodified y*n into temp space + vmov yn_temp, double_n + vcvt.s32.f64 double_n, double_n, #3 + + @ load exp polynomial series constants + vldm literals!, {k8, k7, k6, k5, k4, k3, k2, k1} + + @ mask to extract bit 2 (2^-1 from our fixed-point representation) + vshl.u64 twoto1o2mask, int_1, #2 + + @ make rounding offsets either 1 or -1 instead of 0 or -2 + vorr.u64 ylg2x_round_offset, ylg2x_round_offset, int_1 + vorr.u64 yn_round_offset, yn_round_offset, int_1 + + @ round up to the nearest 1/8th + vadd.s32 ylg2x, ylg2x, ylg2x_round_offset + vadd.s32 double_n, double_n, yn_round_offset + + @ clear out round-up bit for y log2(x) + vbic.s32 ylg2x, ylg2x, int_1 + @ clear out round-up bit for yn + vbic.s32 double_n, double_n, int_1 + @ add together the (fixed precision) rounded parts + vadd.s64 rounded_exponent, double_n, ylg2x + @ turn int_n into a double with value 2^int_n + vshl.i64 int_n, rounded_exponent, #49 + @ compute masks for 2^(1/4) and 2^(1/2) fixups for fractional part of fixed-precision rounded values: + vand.u64 twoto1o4mask, twoto1o4mask, rounded_exponent + vand.u64 twoto1o2mask, twoto1o2mask, rounded_exponent + + @ convert back into floating point, double_n now holds (double) floor(8 y * n + 1)/8 + @ ylg2x now holds (double) floor(8 y * log2(x) + 1)/8 + vcvt.f64.s32 ylg2x, ylg2x, #3 + vcvt.f64.s32 double_n, double_n, #3 + + @ put the 2 bit (0.5) through the roof of twoto1o2mask (make it 0x0 or 0xffffffffffffffff) + vqshl.u64 twoto1o2mask, twoto1o2mask, #62 + @ put the 1 bit (0.25) through the roof of twoto1o4mask (make it 0x0 or 0xffffffffffffffff) + vqshl.u64 twoto1o4mask, twoto1o4mask, #63 + + @ center y*log2(x) fractional part between -0.125 and 0.125 by subtracting (double) floor(8 y * log2(x) + 1)/8 + vsub.f64 ylg2x_temp, ylg2x_temp, ylg2x + @ center y*n fractional part between -0.125 and 0.125 by subtracting (double) floor(8 y * n + 1)/8 + vsub.f64 yn_temp, yn_temp, double_n + + @ Add fractional parts of yn and y log2(x) together + vadd.f64 ss, ylg2x_temp, yn_temp + + @ Result = 1.0 (offset for exp(s) series) + vmov_f64 Result, #0x70 + + @ multiply fractional part of y * log2(x) by ln(2) + vmul.f64 ss, ln2, ss + +@@@@ 10th order polynomial series for Exp(s) @@@@ + + @ ss2 = (ss)^2 + vmul.f64 ss2, ss, ss + + @ twoto1o2mask = twoto1o2mask & twoto1o4 + vand.u64 twoto1o2mask, twoto1o2mask, twoto1o4 + @ twoto1o2mask = twoto1o2mask & twoto1o4 + vand.u64 twoto1o4mask, twoto1o4mask, twoto1o4 + + @ Result = 1.0 + ss + vadd.f64 Result, Result, ss + + @ k7 = ss k8 + k7 + vmla.f64 k7, ss, k8 + + @ ss4 = (ss*ss) * (ss*ss) + vmul.f64 ss4, ss2, ss2 + + @ twoto1o2mask = twoto1o2mask | (double) 1.0 - results in either 1.0 or 2^(1/4) in twoto1o2mask + vorr.u64 twoto1o2mask, twoto1o2mask, double_1 + @ twoto1o2mask = twoto1o4mask | (double) 1.0 - results in either 1.0 or 2^(1/4) in twoto1o4mask + vorr.u64 twoto1o4mask, twoto1o4mask, double_1 + + @ TODO: should setup sign here, expadjustment = 1.0 + vmov_f64 expadjustment, #0x70 + + @ ss3 = (ss*ss) * ss + vmul.f64 ss3, ss2, ss + + @ k0 = 1/2 (first non-unity coefficient) + vmov_f64 k0, #0x60 + + @ Mask out non-exponent bits to make sure we have just 2^int_n + vand.i64 int_n, int_n, EXPONENT_MASK + + @ square twoto1o2mask to get 1.0 or 2^(1/2) + vmul.f64 twoto1o2mask, twoto1o2mask, twoto1o2mask + @ multiply twoto2o4mask into the exponent output adjustment value + vmul.f64 expadjustment, expadjustment, twoto1o4mask + + @ k5 = ss k6 + k5 + vmla.f64 k5, ss, k6 + + @ k3 = ss k4 + k3 + vmla.f64 k3, ss, k4 + + @ k1 = ss k2 + k1 + vmla.f64 k1, ss, k2 + + @ multiply twoto1o2mask into exponent output adjustment value + vmul.f64 expadjustment, expadjustment, twoto1o2mask + + @ k5 = ss^2 ( ss k8 + k7 ) + ss k6 + k5 + vmla.f64 k5, ss2, k7 + + @ k1 = ss^2 ( ss k4 + k3 ) + ss k2 + k1 + vmla.f64 k1, ss2, k3 + + @ Result = 1.0 + ss + 1/2 ss^2 + vmla.f64 Result, ss2, k0 + + @ Adjust int_n so that it's a double precision value that can be multiplied by Result + vadd.i64 expadjustment, int_n, expadjustment + + @ k1 = ss^4 ( ss^2 ( ss k8 + k7 ) + ss k6 + k5 ) + ss^2 ( ss k4 + k3 ) + ss k2 + k1 + vmla.f64 k1, ss4, k5 + + @ Result = 1.0 + ss + 1/2 ss^2 + ss^3 ( ss^4 ( ss^2 ( ss k8 + k7 ) + ss k6 + k5 ) + ss^2 ( ss k4 + k3 ) + ss k2 + k1 ) + vmla.f64 Result, ss3, k1 + + @ multiply by adjustment (sign*(rounding ? sqrt(2) : 1) * 2^int_n) + vmul.f64 Result, expadjustment, Result + +.LleavePow: +#if defined(KRAIT_NO_AAPCS_VFP_MODE) + @ return Result (FP) + vmov Return_lw, Return_hw, Result +#endif +.LleavePowDirect: + @ leave directly returning whatever is in Return_lw and Return_hw + pop {r4, r5, r6, r7, r8, r9, r10, pc} + +.LuseFullImpl: + pop {r4, r5, r6, r7, r8, r9, r10, lr} + b __full_ieee754_pow + +.align 6 +.LliteralTable: +@ Least-sqares tuned constants for 11th order (log2((1+s)/(1-s)): +.LL4: @ ~3/11 + .long 0x53a79915, 0x3fd1b108 +.LL3: @ ~1/3 + .long 0x9ca0567a, 0x3fd554fa +.LL2: @ ~3/7 + .long 0x1408e660, 0x3fdb6db7 +.LL1: @ ~3/5 + .long 0x332D4313, 0x3fe33333 + +@ Least-squares tuned constants for 10th order exp(s): +.LE10: @ ~1/3628800 + .long 0x25c7ba0a, 0x3e92819b +.LE9: @ ~1/362880 + .long 0x9499b49c, 0x3ec72294 +.LE8: @ ~1/40320 + .long 0xabb79d95, 0x3efa019f +.LE7: @ ~1/5040 + .long 0x8723aeaa, 0x3f2a019f +.LE6: @ ~1/720 + .long 0x16c76a94, 0x3f56c16c +.LE5: @ ~1/120 + .long 0x11185da8, 0x3f811111 +.LE4: @ ~1/24 + .long 0x5555551c, 0x3fa55555 +.LE3: @ ~1/6 + .long 0x555554db, 0x3fc55555 + +.LbpA: @ (2^(2/5) - 1) + .long 0x4ee54db1, 0x3fd472d1 + +.LbpB: @ (2^(4/5) - 2^(2/5)) + .long 0x1c8a36cf, 0x3fdafb62 + +.Ltwofifths: @ + .long 0x9999999a, 0x3fd99999 + +.Ltwooverthreeln2: + .long 0xDC3A03FD, 0x3FEEC709 + +.Lln2: @ ln(2) + .long 0xFEFA39EF, 0x3FE62E42 + +.Ltwoto1o4: @ 2^1/4 + .long 0x0a31b715, 0x3ff306fe +END(pow) diff --git a/libm/arm/e_sqrt.S b/libm/arm/e_sqrt.S new file mode 100644 index 000000000..0b5b10c79 --- /dev/null +++ b/libm/arm/e_sqrt.S @@ -0,0 +1,41 @@ +@ Copyright (c) 2013, The Linux Foundation. All rights reserved. +@ +@ Redistribution and use in source and binary forms, with or without +@ modification, are permitted provided that the following conditions are +@ met: +@ * Redistributions of source code must retain the above copyright +@ notice, this list of conditions and the following disclaimer. +@ * Redistributions in binary form must reproduce the above +@ copyright notice, this list of conditions and the following +@ disclaimer in the documentation and/or other materials provided +@ with the distribution. +@ * Neither the name of The Linux Foundation nor the names of its +@ contributors may be used to endorse or promote products derived +@ from this software without specific prior written permission. +@ +@ THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +@ WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +@ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +@ BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +@ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +@ BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +@ OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +@ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include + +ENTRY(sqrt) + vmov.f64 d0, r0, r1 + vsqrt.f64 d0, d0 + vmov.f64 r0, r1, d0 + bx lr +END(sqrt) + +#if (__LDBL_MANT_DIG__ == 53) +.weak sqrtl +.equ sqrtl, sqrt +#endif diff --git a/libm/arm/e_sqrtf.S b/libm/arm/e_sqrtf.S new file mode 100644 index 000000000..b14a77b3e --- /dev/null +++ b/libm/arm/e_sqrtf.S @@ -0,0 +1,36 @@ +@ Copyright (c) 2013, The Linux Foundation. All rights reserved. +@ +@ Redistribution and use in source and binary forms, with or without +@ modification, are permitted provided that the following conditions are +@ met: +@ * Redistributions of source code must retain the above copyright +@ notice, this list of conditions and the following disclaimer. +@ * Redistributions in binary form must reproduce the above +@ copyright notice, this list of conditions and the following +@ disclaimer in the documentation and/or other materials provided +@ with the distribution. +@ * Neither the name of The Linux Foundation nor the names of its +@ contributors may be used to endorse or promote products derived +@ from this software without specific prior written permission. +@ +@ THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +@ WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +@ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +@ BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +@ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +@ BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +@ OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +@ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include + +ENTRY(sqrtf) + vmov.f32 s0, r0 + vsqrt.f32 s0, s0 + vmov.f32 r0, s0 + bx lr +END(sqrtf) diff --git a/libm/arm/s_cos.S b/libm/arm/s_cos.S new file mode 100644 index 000000000..5c8183bb5 --- /dev/null +++ b/libm/arm/s_cos.S @@ -0,0 +1,427 @@ +@ Copyright (c) 2012, The Linux Foundation. All rights reserved. +@ +@ Redistribution and use in source and binary forms, with or without +@ modification, are permitted provided that the following conditions are +@ met: +@ * Redistributions of source code must retain the above copyright +@ notice, this list of conditions and the following disclaimer. +@ * Redistributions in binary form must reproduce the above +@ copyright notice, this list of conditions and the following +@ disclaimer in the documentation and/or other materials provided +@ with the distribution. +@ * Neither the name of The Linux Foundation nor the names of its +@ contributors may be used to endorse or promote products derived +@ from this software without specific prior written permission. +@ +@ THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +@ WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +@ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +@ BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +@ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +@ BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +@ OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +@ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +@ +@ Additional notices preserved for attributions purposes only. +@ +@ ==================================================== +@ Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +@ +@ Developed at SunSoft, a Sun Microsystems, Inc. business. +@ Permission to use, copy, modify, and distribute this +@ software is freely granted, provided that this notice +@ is preserved. +@ ==================================================== +@ +@ ==================================================== +@ Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +@ +@ Developed at SunPro, a Sun Microsystems, Inc. business. +@ Permission to use, copy, modify, and distribute this +@ software is freely granted, provided that this notice +@ is preserved. +@ ==================================================== + +#include +#include + +#define vmov_f64 fconstd + +ENTRY(cos) + push {r4, r6, r7, lr} + vmov d0, r0, r1 + mov r2, r0 + mov r3, r1 + movw r1, #0x21fb + movt r1, #0x3fe9 + mov r4, r3 + bic r3, r3, #0x80000000 + sub sp, sp, #48 + cmp r3, r1 + bgt .Lxgtpio4 + cmp r3, #0x3e400000 + bge .Lxnottiny + vcvt.s32.f64 s15, d0 + vmov r3, s15 + cmp r3, #0 + beq .Lreturnone +.Lxnottiny: + vmov.i64 d1, #0 + bl __kernel_cos +.Lleave_cos: + vmov r0, r1, d0 +.Lleave_cos_direct: + add sp, sp, #48 + pop {r4, r6, r7, pc} +.Lxgtpio4: + movw r2, #0xffff + movt r2, #0x7fef + cmp r3, r2 + bgt .LxisNaN + movw r0, #0xd97b + movt r0, #0x4002 + cmp r3, r0 + movw r2, #0x21fb + bgt .Lxge3pio4 + cmp r4, #0 + movt r2, #0x3ff9 + ble .Lsmallxisnegative + vldr d16, .Lpio2_1 + cmp r3, r2 + vsub.f64 d16, d0, d16 + beq .Lxnearpio2 + vldr d17, .Lpio2_1t +.Lfinalizesmallxremainder: + vsub.f64 d0, d16, d17 + vsub.f64 d16, d16, d0 + vstr d0, [sp, #8] + vsub.f64 d1, d16, d17 + vstr d1, [sp, #16] +.Lnmod3is1: + mov r0, #1 + bl __kernel_sin + vneg.f64 d0, d0 + b .Lleave_cos +.Lreturnone: + mov r0, #0 + movw r1, #0x0000 + movt r1, #0x3ff0 + vmov_f64 d0, #0x70 + b .Lleave_cos_direct +.LxisNaN: + vsub.f64 d0, d0, d0 + b .Lleave_cos +.Lxge3pio4: + movt r2, #0x4139 + cmp r3, r2 + bgt .Lxgigantic + vmov_f64 d3, #0x60 + vldr d2, .Linvpio2 + vldr d18, .Lpio2_1 + vabs.f64 d16, d0 + vmla.f64 d3, d16, d2 + vcvt.s32.f64 s3, d3 + vcvt.f64.s32 d17, s3 + vmov r0, s3 + cmp r0, #31 + vmls.f64 d16, d17, d18 + vldr d18, .Lpio2_1t + vmul.f64 d18, d17, d18 + bgt .Lcomputeremainder + ldr r2, .Lnpio2_hw_ptr + sub lr, r0, #1 +.LPICnpio2_hw0: + add r12, pc, r2 + ldr r1, [r12, lr, lsl #2] + cmp r3, r1 + beq .Lcomputeremainder +.Lfinishthirditeration: + vsub.f64 d0, d16, d18 + vstr d0, [sp, #8] +.Lfinishcomputingremainder: + vsub.f64 d16, d16, d0 + cmp r4, #0 + vsub.f64 d1, d16, d18 + vstr d1, [sp, #16] + blt .Lhandlenegativex +.Lselectregion: + and r0, r0, #3 + cmp r0, #1 + beq .Lnmod3is1 + cmp r0, #2 + beq .Lnmod3is2 + cmp r0, #0 + bne .Lnmod3is0 + bl __kernel_cos + b .Lleave_cos +.Lxgigantic: + asr r2, r3, #20 + vmov r6, r7, d0 + sub r2, r2, #1040 + mov r0, r6 + sub r2, r2, #6 + vldr d16, .Ltwo24 + sub r1, r3, r2, lsl #20 + vmov d18, r0, r1 + vcvt.s32.f64 s15, d18 + add r1, sp, #48 + mov r3, #3 + vcvt.f64.s32 d17, s15 + vsub.f64 d18, d18, d17 + vstr d17, [sp, #24] + vmul.f64 d18, d18, d16 + vcvt.s32.f64 s15, d18 + vcvt.f64.s32 d17, s15 + vsub.f64 d18, d18, d17 + vstr d17, [sp, #32] + vmul.f64 d16, d18, d16 + fcmpzd d16 + vstmdb r1!, {d16} + vmrs APSR_nzcv, fpscr + bne .Lprocessnonzeroterm +.Lskipzeroterms: + vldmdb r1!, {d16} + sub r3, r3, #1 + fcmpzd d16 + vmrs APSR_nzcv, fpscr + beq .Lskipzeroterms +.Lprocessnonzeroterm: + ldr r12, .Ltwo_over_pi_ptr + add r0, sp, #24 + add r1, sp, #8 +.LPICtwo_over_pi0: + add lr, pc, r12 + mov r12, #2 + str lr, [sp, #4] + str r12, [sp] + bl __kernel_rem_pio2 + cmp r4, #0 + vldr d0, [sp, #8] + blt .Lhandlenegativxalso + vldr d1, [sp, #16] + b .Lselectregion +.Lxnearpio2: + vldr d17, .Lpio2_2 + vsub.f64 d16, d16, d17 + vldr d17, .Lpio2_2t + b .Lfinalizesmallxremainder +.Lsmallxisnegative: + vldr d1, .Lpio2_1 + cmp r3, r2 + vadd.f64 d16, d0, d1 + beq .Lxnearnegpio2 + vldr d17, .Lpio2_1t +.Lfinalizesmallnegxremainder: + vadd.f64 d0, d16, d17 + vsub.f64 d16, d16, d0 + vstr d0, [sp, #8] + vadd.f64 d1, d16, d17 + vstr d1, [sp, #16] +.Lnmod3is0: + mov r0, #1 + bl __kernel_sin + b .Lleave_cos +.Lnmod3is2: + bl __kernel_cos + vneg.f64 d0, d0 + b .Lleave_cos +.Lcomputeremainder: + vsub.f64 d0, d16, d18 + asr r1, r3, #20 + vmov r2, r3, d0 + ubfx r3, r3, #20, #11 + rsb r3, r3, r1 + vstr d0, [sp, #8] + cmp r3, #16 + ble .Lfinishcomputingremainder + vldr d18, .Lpio2_2 + vmul.f64 d20, d17, d18 + vsub.f64 d19, d16, d20 + vsub.f64 d16, d16, d19 + vsub.f64 d18, d16, d20 + vldr d16, .Lpio2_2t + vnmls.f64 d18, d17, d16 + vsub.f64 d0, d19, d18 + vmov r2, r3, d0 + ubfx r3, r3, #20, #11 + rsb r1, r3, r1 + vstr d0, [sp, #8] + cmp r1, #49 + ble .Lfinishseconditeration + vldr d5, .Lpio2_3 + vmul.f64 d20, d17, d5 + vsub.f64 d16, d19, d20 + vsub.f64 d4, d19, d16 + vldr d19, .Lpio2_3t + vsub.f64 d18, d4, d20 + vnmls.f64 d18, d17, d19 + b .Lfinishthirditeration +.Lhandlenegativex: + vneg.f64 d0, d0 + rsb r0, r0, #0 + vneg.f64 d1, d1 + vstr d0, [sp, #8] + vstr d1, [sp, #16] + b .Lselectregion +.Lfinishseconditeration: + vmov d16, d19 + b .Lfinishcomputingremainder +.Lxnearnegpio2: + vldr d0, .Lpio2_2 + vldr d17, .Lpio2_2t + vadd.f64 d16, d16, d0 + b .Lfinalizesmallnegxremainder +.Lhandlenegativxalso: + vldr d6, [sp, #16] + vneg.f64 d0, d0 + rsb r0, r0, #0 + vneg.f64 d1, d6 + vstr d0, [sp, #8] + vstr d1, [sp, #16] + b .Lselectregion + +.align 3 +.Lpio2_1: + .word 0x54400000, 0x3ff921fb +.Lpio2_1t: + .word 0x1a626331, 0x3dd0b461 +.Linvpio2: + .word 0x6dc9c883, 0x3fe45f30 +.Ltwo24: + .word 0x00000000, 0x41700000 +.Lpio2_2: + .word 0x1a600000, 0x3dd0b461 +.Lpio2_2t: + .word 0x2e037073, 0x3ba3198a +.Lpio2_3: + .word 0x2e000000, 0x3ba3198a +.Lpio2_3t: + .word 0x252049c1, 0x397b839a +.Lnpio2_hw_ptr: + .word .Lnpio2_hw-(.LPICnpio2_hw0+8) +.Ltwo_over_pi_ptr: + .word .Ltwo_over_pi-(.LPICtwo_over_pi0+8) +END(cos) + +#if (__LDBL_MANT_DIG__ == 53) +.weak cosl +.equ cosl, cos +#else +ENTRY(cosl) +END(cosl) +#endif + + .section .rodata.npio2_hw,"a",%progbits + .align 2 +.Lnpio2_hw = . + 0 + .type npio2_hw, %object + .size npio2_hw, 128 +npio2_hw: + .word 0x3ff921fb + .word 0x400921fb + .word 0x4012d97c + .word 0x401921fb + .word 0x401f6a7a + .word 0x4022d97c + .word 0x4025fdbb + .word 0x402921fb + .word 0x402c463a + .word 0x402f6a7a + .word 0x4031475c + .word 0x4032d97c + .word 0x40346b9c + .word 0x4035fdbb + .word 0x40378fdb + .word 0x403921fb + .word 0x403ab41b + .word 0x403c463a + .word 0x403dd85a + .word 0x403f6a7a + .word 0x40407e4c + .word 0x4041475c + .word 0x4042106c + .word 0x4042d97c + .word 0x4043a28c + .word 0x40446b9c + .word 0x404534ac + .word 0x4045fdbb + .word 0x4046c6cb + .word 0x40478fdb + .word 0x404858eb + .word 0x404921fb + + .section .rodata.two_over_pi,"a",%progbits + .align 2 +.Ltwo_over_pi = . + 0 + .type two_over_pi, %object + .size two_over_pi, 264 +two_over_pi: + .word 0x00a2f983 + .word 0x006e4e44 + .word 0x001529fc + .word 0x002757d1 + .word 0x00f534dd + .word 0x00c0db62 + .word 0x0095993c + .word 0x00439041 + .word 0x00fe5163 + .word 0x00abdebb + .word 0x00c561b7 + .word 0x00246e3a + .word 0x00424dd2 + .word 0x00e00649 + .word 0x002eea09 + .word 0x00d1921c + .word 0x00fe1deb + .word 0x001cb129 + .word 0x00a73ee8 + .word 0x008235f5 + .word 0x002ebb44 + .word 0x0084e99c + .word 0x007026b4 + .word 0x005f7e41 + .word 0x003991d6 + .word 0x00398353 + .word 0x0039f49c + .word 0x00845f8b + .word 0x00bdf928 + .word 0x003b1ff8 + .word 0x0097ffde + .word 0x0005980f + .word 0x00ef2f11 + .word 0x008b5a0a + .word 0x006d1f6d + .word 0x00367ecf + .word 0x0027cb09 + .word 0x00b74f46 + .word 0x003f669e + .word 0x005fea2d + .word 0x007527ba + .word 0x00c7ebe5 + .word 0x00f17b3d + .word 0x000739f7 + .word 0x008a5292 + .word 0x00ea6bfb + .word 0x005fb11f + .word 0x008d5d08 + .word 0x00560330 + .word 0x0046fc7b + .word 0x006babf0 + .word 0x00cfbc20 + .word 0x009af436 + .word 0x001da9e3 + .word 0x0091615e + .word 0x00e61b08 + .word 0x00659985 + .word 0x005f14a0 + .word 0x0068408d + .word 0x00ffd880 + .word 0x004d7327 + .word 0x00310606 + .word 0x001556ca + .word 0x0073a8c9 + .word 0x0060e27b + .word 0x00c08c6b diff --git a/libm/arm/s_floor.S b/libm/arm/s_floor.S new file mode 100644 index 000000000..44053581e --- /dev/null +++ b/libm/arm/s_floor.S @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2013-2014, NVIDIA Corporation. All rights reserved. + * Johnny Qiu + * Shu Zhang + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of The Linux Foundation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +ENTRY(floor) /* x in r0, r1 */ + + and r3, r1, #0x80000000 /* sign(x) */ + bic r1, r1, #0x80000000 /* x = abs(x) */ + + /* extract exp of x */ + lsr r2, r1, #20 + sub r2, r2, #0x3fc + subs r2, r2, #0x3 /* r2 <- exp */ + + /* |x| < 1.0? */ + blt .Lx_lt_one + + /* x < 0? */ + cmp r3, #0 + bne .Lclr_frac_neg + + /* |x| <= 2^20? */ + cmp r2, #20 + ble .Lclr_frac_r1 + + /* |x| < 2^52? */ + cmp r2, #52 + blt .Lclr_frac_r0 + + /* return x */ + bx lr + +.Lclr_frac_r1: + rsb r2, r2, #20 + lsr r1, r1, r2 + lsl r1, r1, r2 + mov r0, #0 + bx lr + +.Lclr_frac_r0: + rsb r2, r2, #52 + lsr r0, r0, r2 + lsl r0, r0, r2 + bx lr + +.Lclr_frac_neg: + /* |x| <= 2^20? */ + cmp r2, #20 + ble .Lclr_frac_r1_neg + + /* |x| < 2^52? */ + cmp r2, #52 + blt .Lclr_frac_r0_neg + + /* return x */ + orr r1, r1, #0x80000000 + bx lr + +.Lclr_frac_r1_neg: + rsb r2, r2, #20 + mov r3, #1 + lsl r3, r3, r2 + sub r3, r3, #1 + and r3, r1, r3 + orr r3, r3, r0 + lsr r1, r1, r2 + lsl r1, r1, r2 + mov r0, #0 + b .Lreturn_x_neg + +.Lclr_frac_r0_neg: + rsb r2, r2, #52 + mov r3, #1 + lsl r3, r3, r2 + sub r3, r3, #1 + and r3, r0, r3 + lsr r0, r0, r2 + lsl r0, r0, r2 + b .Lreturn_x_neg + +.Lx_lt_one: + /* x == +-0? */ + cmp r0, #0 + cmpeq r1, #0 + orreq r1, r1, r3 + bxeq lr + + /* (x > 0) ? 0 : -1 */ + mov r1, #0x00100000 + mov r0, #0 + cmp r3, #0 + movne r1, #0xc0000000 + sub r1, r1, #0x00100000 + bx lr + +.Lreturn_x_neg: + cmp r3, #0 + orr r1, r1, #0x80000000 + bxeq lr + + vmov d16, r0, r1 + vmov.f64 d18, #1.0 + vsub.f64 d16, d16, d18 + vmov r0, r1, d16 + bx lr + +END(floor) + +#if LDBL_MANT_DIG == 53 + .weak floorl + .equ floorl,floor +#endif diff --git a/libm/arm/s_sin.S b/libm/arm/s_sin.S new file mode 100644 index 000000000..8b9eae4ed --- /dev/null +++ b/libm/arm/s_sin.S @@ -0,0 +1,422 @@ +@ Copyright (c) 2012, The Linux Foundation. All rights reserved. +@ +@ Redistribution and use in source and binary forms, with or without +@ modification, are permitted provided that the following conditions are +@ met: +@ * Redistributions of source code must retain the above copyright +@ notice, this list of conditions and the following disclaimer. +@ * Redistributions in binary form must reproduce the above +@ copyright notice, this list of conditions and the following +@ disclaimer in the documentation and/or other materials provided +@ with the distribution. +@ * Neither the name of The Linux Foundation nor the names of its +@ contributors may be used to endorse or promote products derived +@ from this software without specific prior written permission. +@ +@ THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +@ WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +@ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +@ BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +@ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +@ BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +@ OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +@ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +@ +@ Additional notices preserved for attributions purposes only. +@ +@ ==================================================== +@ Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +@ +@ Developed at SunSoft, a Sun Microsystems, Inc. business. +@ Permission to use, copy, modify, and distribute this +@ software is freely granted, provided that this notice +@ is preserved. +@ ==================================================== +@ +@ ==================================================== +@ Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +@ +@ Developed at SunPro, a Sun Microsystems, Inc. business. +@ Permission to use, copy, modify, and distribute this +@ software is freely granted, provided that this notice +@ is preserved. +@ ==================================================== + +#include +#include + +#define vmov_f64 fconstd + +ENTRY(sin) + push {r4, r6, r7, lr} + vmov d0, r0, r1 + mov r2, r0 + mov r3, r1 + movw r1, #0x21fb + movt r1, #0x3fe9 + mov r4, r3 + bic r3, r3, #0x80000000 + sub sp, sp, #48 + cmp r3, r1 + bgt .Lxgtpio4 + cmp r3, #0x3e400000 + bge .Lxnottiny + vcvt.s32.f64 s15, d0 + vmov r3, s15 + cmp r3, #0 + bne .Lxnottiny +.Lleave_sin: + vmov r0, r1, d0 + add sp, sp, #48 + pop {r4, r6, r7, pc} +.Lxgtpio4: + movw r2, #0xffff + movt r2, #0x7fef + cmp r3, r2 + bgt .LxisNaN + movw r0, #0xd97b + movt r0, #0x4002 + cmp r3, r0 + movw r2, #0x21fb + bgt .Lxge3pio4 + cmp r4, #0 + movt r2, #0x3ff9 + ble .Lsmallxisnegative + vldr d16, .Lpio2_1 + cmp r3, r2 + vsub.f64 d16, d0, d16 + beq .Lxnearpio2 + vldr d17, .Lpio2_1t +.Lfinalizesmallxremainder: + vsub.f64 d0, d16, d17 + vsub.f64 d16, d16, d0 + vstr d0, [sp, #8] + vsub.f64 d1, d16, d17 + vstr d1, [sp, #16] +.Lnmod3is1: + bl __kernel_cos + b .Lleave_sin +.Lxnottiny: + vmov.i64 d1, #0 + mov r0, #0 + bl __kernel_sin + b .Lleave_sin +.LxisNaN: + vsub.f64 d0, d0, d0 + b .Lleave_sin +.Lxge3pio4: + movt r2, #0x4139 + cmp r3, r2 + bgt .Lxgigantic + vmov_f64 d3, #0x60 + vldr d2, .Linvpio2 + vldr d18, .Lpio2_1 + vabs.f64 d16, d0 + vmla.f64 d3, d16, d2 + vcvt.s32.f64 s3, d3 + vcvt.f64.s32 d17, s3 + vmov r0, s3 + cmp r0, #31 + vmls.f64 d16, d17, d18 + vldr d18, .Lpio2_1t + vmul.f64 d18, d17, d18 + bgt .Lcomputeremainder + ldr r2, .Lnpio2_hw_ptr + sub lr, r0, #1 +.LPICnpio2_hw0: + add r12, pc, r2 + ldr r1, [r12, lr, lsl #2] + cmp r3, r1 + beq .Lcomputeremainder +.Lfinishthirditeration: + vsub.f64 d0, d16, d18 + vstr d0, [sp, #8] +.Lfinishcomputingremainder: + vsub.f64 d16, d16, d0 + cmp r4, #0 + vsub.f64 d1, d16, d18 + vstr d1, [sp, #16] + blt .Lhandlenegativex +.Lselectregion: + and r0, r0, #3 + cmp r0, #1 + beq .Lnmod3is1 + cmp r0, #2 + beq .Lnmod3is2 + cmp r0, #0 + bne .Lnmod3is0 + mov r0, #1 + bl __kernel_sin + b .Lleave_sin +.Lxgigantic: + asr r2, r3, #20 + vmov r6, r7, d0 + sub r2, r2, #1040 + mov r0, r6 + sub r2, r2, #6 + vldr d16, .Ltwo24 + sub r1, r3, r2, lsl #20 + vmov d18, r0, r1 + vcvt.s32.f64 s15, d18 + add r1, sp, #48 + mov r3, #3 + vcvt.f64.s32 d17, s15 + vsub.f64 d18, d18, d17 + vstr d17, [sp, #24] + vmul.f64 d18, d18, d16 + vcvt.s32.f64 s15, d18 + vcvt.f64.s32 d17, s15 + vsub.f64 d18, d18, d17 + vstr d17, [sp, #32] + vmul.f64 d16, d18, d16 + fcmpzd d16 + vstmdb r1!, {d16} + vmrs APSR_nzcv, fpscr + bne .Lprocessnonzeroterm +.Lskipzeroterms: + vldmdb r1!, {d16} + sub r3, r3, #1 + fcmpzd d16 + vmrs APSR_nzcv, fpscr + beq .Lskipzeroterms +.Lprocessnonzeroterm: + ldr r12, .Ltwo_over_pi_ptr + add r0, sp, #24 + add r1, sp, #8 +.LPICtwo_over_pi0: + add lr, pc, r12 + mov r12, #2 + str lr, [sp, #4] + str r12, [sp] + bl __kernel_rem_pio2 + cmp r4, #0 + vldr d0, [sp, #8] + blt .Lhandlenegativexalso + vldr d1, [sp, #16] + b .Lselectregion +.Lxnearpio2: + vldr d17, .Lpio2_2 + vsub.f64 d16, d16, d17 + vldr d17, .Lpio2_2t + b .Lfinalizesmallxremainder +.Lsmallxisnegative: + vldr d1, .Lpio2_1 + cmp r3, r2 + vadd.f64 d16, d0, d1 + beq .Lxnearnegpio2 + vldr d17, .Lpio2_1t +.Lfinalizesmallnegxremainder: + vadd.f64 d0, d16, d17 + vsub.f64 d16, d16, d0 + vstr d0, [sp, #8] + vadd.f64 d1, d16, d17 + vstr d1, [sp, #16] +.Lnmod3is0: + bl __kernel_cos + vneg.f64 d0, d0 + b .Lleave_sin +.Lnmod3is2: + mov r0, #1 + bl __kernel_sin + vneg.f64 d0, d0 + b .Lleave_sin +.Lcomputeremainder: + vsub.f64 d0, d16, d18 + asr r1, r3, #20 + vmov r2, r3, d0 + ubfx r3, r3, #20, #11 + rsb r3, r3, r1 + vstr d0, [sp, #8] + cmp r3, #16 + ble .Lfinishcomputingremainder + vldr d18, .Lpio2_2 + vmul.f64 d20, d17, d18 + vsub.f64 d19, d16, d20 + vsub.f64 d16, d16, d19 + vsub.f64 d18, d16, d20 + vldr d16, .Lpio2_2t + vnmls.f64 d18, d17, d16 + vsub.f64 d0, d19, d18 + vmov r2, r3, d0 + ubfx r3, r3, #20, #11 + rsb r1, r3, r1 + vstr d0, [sp, #8] + cmp r1, #49 + ble .Lfinishseconditeration + vldr d5, .Lpio2_3 + vmul.f64 d20, d17, d5 + vsub.f64 d16, d19, d20 + vsub.f64 d4, d19, d16 + vldr d19, .Lpio2_3t + vsub.f64 d18, d4, d20 + vnmls.f64 d18, d17, d19 + b .Lfinishthirditeration +.Lhandlenegativex: + vneg.f64 d0, d0 + rsb r0, r0, #0 + vneg.f64 d1, d1 + vstr d0, [sp, #8] + vstr d1, [sp, #16] + b .Lselectregion +.Lfinishseconditeration: + vmov d16, d19 + b .Lfinishcomputingremainder +.Lxnearnegpio2: + vldr d0, .Lpio2_2 + vldr d17, .Lpio2_2t + vadd.f64 d16, d16, d0 + b .Lfinalizesmallnegxremainder +.Lhandlenegativexalso: + vldr d6, [sp, #16] + vneg.f64 d0, d0 + rsb r0, r0, #0 + vneg.f64 d1, d6 + vstr d0, [sp, #8] + vstr d1, [sp, #16] + b .Lselectregion + +.align 3 +.Lpio2_1: + .word 0x54400000, 0x3ff921fb +.Lpio2_1t: + .word 0x1a626331, 0x3dd0b461 +.Linvpio2: + .word 0x6dc9c883, 0x3fe45f30 +.Ltwo24: + .word 0x00000000, 0x41700000 +.Lpio2_2: + .word 0x1a600000, 0x3dd0b461 +.Lpio2_2t: + .word 0x2e037073, 0x3ba3198a +.Lpio2_3: + .word 0x2e000000, 0x3ba3198a +.Lpio2_3t: + .word 0x252049c1, 0x397b839a +.Lnpio2_hw_ptr: + .word .Lnpio2_hw-(.LPICnpio2_hw0+8) +.Ltwo_over_pi_ptr: + .word .Ltwo_over_pi-(.LPICtwo_over_pi0+8) +END(sin) + +#if (__LDBL_MANT_DIG__ == 53) +.weak sinl +.equ sinl, sin +#else +ENTRY(sinl) +END(sinl) +#endif + + .section .rodata.npio2_hw,"a",%progbits + .align 2 +.Lnpio2_hw = . + 0 + .type npio2_hw, %object + .size npio2_hw, 128 +npio2_hw: + .word 0x3ff921fb + .word 0x400921fb + .word 0x4012d97c + .word 0x401921fb + .word 0x401f6a7a + .word 0x4022d97c + .word 0x4025fdbb + .word 0x402921fb + .word 0x402c463a + .word 0x402f6a7a + .word 0x4031475c + .word 0x4032d97c + .word 0x40346b9c + .word 0x4035fdbb + .word 0x40378fdb + .word 0x403921fb + .word 0x403ab41b + .word 0x403c463a + .word 0x403dd85a + .word 0x403f6a7a + .word 0x40407e4c + .word 0x4041475c + .word 0x4042106c + .word 0x4042d97c + .word 0x4043a28c + .word 0x40446b9c + .word 0x404534ac + .word 0x4045fdbb + .word 0x4046c6cb + .word 0x40478fdb + .word 0x404858eb + .word 0x404921fb + + .section .rodata.two_over_pi,"a",%progbits + .align 2 +.Ltwo_over_pi = . + 0 + .type two_over_pi, %object + .size two_over_pi, 264 +two_over_pi: + .word 0x00a2f983 + .word 0x006e4e44 + .word 0x001529fc + .word 0x002757d1 + .word 0x00f534dd + .word 0x00c0db62 + .word 0x0095993c + .word 0x00439041 + .word 0x00fe5163 + .word 0x00abdebb + .word 0x00c561b7 + .word 0x00246e3a + .word 0x00424dd2 + .word 0x00e00649 + .word 0x002eea09 + .word 0x00d1921c + .word 0x00fe1deb + .word 0x001cb129 + .word 0x00a73ee8 + .word 0x008235f5 + .word 0x002ebb44 + .word 0x0084e99c + .word 0x007026b4 + .word 0x005f7e41 + .word 0x003991d6 + .word 0x00398353 + .word 0x0039f49c + .word 0x00845f8b + .word 0x00bdf928 + .word 0x003b1ff8 + .word 0x0097ffde + .word 0x0005980f + .word 0x00ef2f11 + .word 0x008b5a0a + .word 0x006d1f6d + .word 0x00367ecf + .word 0x0027cb09 + .word 0x00b74f46 + .word 0x003f669e + .word 0x005fea2d + .word 0x007527ba + .word 0x00c7ebe5 + .word 0x00f17b3d + .word 0x000739f7 + .word 0x008a5292 + .word 0x00ea6bfb + .word 0x005fb11f + .word 0x008d5d08 + .word 0x00560330 + .word 0x0046fc7b + .word 0x006babf0 + .word 0x00cfbc20 + .word 0x009af436 + .word 0x001da9e3 + .word 0x0091615e + .word 0x00e61b08 + .word 0x00659985 + .word 0x005f14a0 + .word 0x0068408d + .word 0x00ffd880 + .word 0x004d7327 + .word 0x00310606 + .word 0x001556ca + .word 0x0073a8c9 + .word 0x0060e27b + .word 0x00c08c6b diff --git a/libm/arm64/ceil.S b/libm/arm64/ceil.S new file mode 100644 index 000000000..006c9887a --- /dev/null +++ b/libm/arm64/ceil.S @@ -0,0 +1,27 @@ +/* + * Copyright 2015, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +ENTRY(ceil) + frintP d0, d0 + ret +END(ceil) + +ENTRY(ceilf) + frintP s0, s0 + ret +END(ceilf) diff --git a/libm/arm64/e_pow64.S b/libm/arm64/e_pow64.S new file mode 100644 index 000000000..2a30c4aaf --- /dev/null +++ b/libm/arm64/e_pow64.S @@ -0,0 +1,450 @@ +/* Copyright (c) 2009-2014 The Linux Foundation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Linux Foundation nor the names of its contributors may + * be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +//#define NO_FUSED_MULTIPLY + +#define KRAIT_NO_AAPCS_VFP_MODE + +ENTRY(pow) +#if defined(KRAIT_NO_AAPCS_VFP_MODE) + // ARM ABI has inputs coming in via d registers, lets copy to x registers + fmov x0, d0 + fmov x1, d1 +#endif + mov w12, #0x40100000 // high word of 64-bit 4.0 + + // pre-staged bp values + ldr d5, .LbpA + ldr d3, .LbpB + + // load two fifths into constant term in case we need it due to offsets + ldr d6, .Ltwofifths + + // bp is initially 1.0, may adjust later based on x value + fmov d4, #1.0 + + // twoto1o5 = 2^(1/5) (input bracketing) + ldr x4, .Ltwoto1o5 + + // twoto3o5 = 2^(3/5) (input bracketing) + ldr x5, .Ltwoto3o5 + + // extract xmantissa + bic x6, x0, #0xFFF0000000000000 + + // begin preparing a mask for normalization (high 32-bit mask) + movi d31, #0xFFFFFFFF00000000 + + // double_1 = (double) 1.0 + fmov d28, #1.0 + + cmp x6, x4 + + shl d30, d31, #20 // d30 can mask just sign/exp bits + ushr d29, d31, #63 // mask has only bit 1 set + + adr x10, .LliteralTable // x10->k4 in literal table (below) + + bhi .Lxgt2to1over5 + // zero out lg2 constant term if don't offset our input + fsub d6, d6, d6 + b .Lxle2to1over5 + +.Lxgt2to1over5: + // if normalized x > 2^(1/5), bp = 1 + (2^(2/5)-1) = 2^(2/5) + fadd d4, d4, d5 + +.Lxle2to1over5: + ldr d5, .Lln2 // d5 = ln2 = 0.69314718056 + + cmp x6, x5 // non-normalized compare + +//@@@ X Value Normalization @@@@ + + // ss = abs(x) 2^(-1024) + bic v16.8B, v0.8B, v30.8B // mantissa of x into v16.8B (aka d16) + + // N = (floor(log2(x)) + 0x3ff) * 2^52 + and v2.8B, v0.8B, v30.8B // exponent of x (d0) into v2.8B aka d2 + + bls .Lxle2to3over5 // branch not taken if (x6 > x5) + // if normalized x > 2^(3/5), bp = 2^(2/5) + (2^(4/5) - 2^(2/5)) = 2^(4/5) + fadd d4, d4, d3 // d4 = 2^(2/5) + (2^(4/5) - 2^(2/5)) = 2^(4/5) + fadd d6, d6, d6 // d6 = 2/5 + 2/5 = 4/5 or 0 (see logic above) + +.Lxle2to3over5: + + lsr x2, x0, #32 // Need just high word of x...x2 can take it + cmp w2, w12 // Compare x to 4.0 (high word only) + lsr x3, x1, #32 // Need just high word of y...x3 can take it. + ccmp w3, w12, #2, ls // If x < 4.0, compare y to 4.0 (high word) + bic w12, w12, #0xFFFF0000 // Change w12 for compare to 0.0000325 + orr w12, w12, #0x3fe00000 + ccmp w12, w2, #2, ls // If y < 4, compare 0.5 to x + + // load log2 polynomial series constants + ldp d24, d25, [x10, #0] + ldp d26, d27, [x10, #16] + + // s = abs(x) 2^(-floor(log2(x))) (normalize abs(x) to around 1) + orr v16.8B, v16.8B, v28.8B + +//@@@ 3/2 (Log(bp(1+s)/(1-s))) input computation (s = (x-bp)/(x+bp)) @@@@ + + fsub d19, d16, d4 // take normalized x and subtract 2^(4/5) from it + fadd d20, d16, d4 + bhi .LuseFullImpl // |x| < 0.5 or x > 4 or y > 4 + + // s = (x-1)/(x+1) + fdiv d16, d19, d20 + + // load 2/(3log2) into lg2coeff + ldr d21, .Ltwooverthreeln2 + + // N = floor(log2(x)) * 2^52 + sub d2, d2, d28 + +//@@@ 3/2 (Log(bp(1+s)/(1-s))) polynomial series @@@@ + + // ss2 = ((x-bp)/(x+bp))^2 + fmul d17, d16, d16 + + // ylg2x = 3.0 + fmov d0, #3.0 + fmul d18, d17, d17 + + // todo: useful later for two-way clamp + fmul d21, d21, d1 + + // N = floor(log2(x)) + sshr d2, d2, #52 + // k3 = ss^2 * L4 + L3 +#ifdef NO_FUSED_MULTIPLY + fmul d3, d17, v24.2D[0] + fadd d25, d25, d3 + + // k1 = ss^2 * L2 + L1 + fmul d3, d17, v26.2D[0] + fadd d27, d27, d3 +#else + fmla d25, d17, v24.2D[0] + + // k1 = ss^2 * L2 + L1 + fmla d27, d17, v26.2D[0] +#endif + + // scale ss by 2/(3 ln 2) + fmul d21, d16, d21 + + // ylg2x = 3.0 + s^2 + fadd d0, d0, d17 + + fmov x2, d2 + scvtf d3, w2 // Low-order 32-bit integer half of d2 to fp64 + + // k1 = s^4 (s^2 L4 + L3) + s^2 L2 + L1 +#ifdef NO_FUSED_MULTIPLY + fmul d31, d18, v25.2D[0] + fadd d27, d27, d31 +#else + fmla d27, d18, v25.2D[0] +#endif + // add in constant term + fadd d3, d3, d6 + + // ylg2x = 3.0 + s^2 + s^4 (s^4 (s^2 L4 + L3) + s^2 L2 + L1) +#ifdef NO_FUSED_MULTIPLY + fmul d31, d18, v27.2D[0] + fadd d0, d0, d31 +#else + fmla d0, d18, v27.2D[0] +#endif + // ylg2x = y 2 s / (3 ln(2)) (3.0 + s^2 + s^4 (s^4(s^2 L4 + L3) + s^2 L2 + L1) + fmul d0, d21, d0 + +//@@@ Compute input to Exp(s) (s = y(n + log2(x)) - (floor(8 yn + 1)/8 + floor(8 ylog2(x) + 1)/8) @@@@@ + + // mask to extract bit 1 (2^-2 from our fixed-point representation) + shl d4, d29, #1 + + // double_n = y * n + fmul d3, d3, d1 + + // Load 2^(1/4) for later computations + ldr d6, .Ltwoto1o4 + + // either add or subtract one based on the sign of double_n and ylg2x + sshr d16, d0, #62 + sshr d19, d3, #62 + + // move unmodified y*lg2x into temp space + fmov d17, d0 + + // compute floor(8 y * n + 1)/8 + // and floor(8 y (log2(x)) + 1)/8 + fcvtzs w2, d0, #3 // no instruction exists to use s0 as a direct target + fmov s0, w2 // run our conversion into w2, then mov it to compensate + // move unmodified y*n into temp space + fmov d18, d3 + fcvtzs w2, d3, #3 + fmov s3, w2 + + // load exp polynomial series constants + ldp d20, d21, [x10, #32] + ldp d22, d23, [x10, #48] + ldp d24, d25, [x10, #64] + ldp d26, d27, [x10, #80] + + // mask to extract bit 2 (2^-1 from our fixed-point representation) + shl d1, d29, #2 + + // make rounding offsets either 1 or -1 instead of 0 or -2 + orr v16.8B, v16.8B, v29.8B + orr v19.8B, v19.8B, v29.8B + + // round up to the nearest 1/8th + add d0, d0, d16 + add d3, d3, d19 + + // clear out round-up bit for y log2(x) + bic v0.8B, v0.8B, v29.8B + // clear out round-up bit for yn + bic v3.8B, v3.8B, v29.8B + // add together the (fixed precision) rounded parts + add d31, d3, d0 + // turn int_n into a double with value 2^int_n + shl d2, d31, #49 + // compute masks for 2^(1/4) and 2^(1/2) fixups for fractional part of fixed-precision rounded values: + and v4.8B, v4.8B, v31.8B + and v1.8B, v1.8B, v31.8B + + // convert back into floating point, d3 now holds (double) floor(8 y * n + 1)/8 + // d0 now holds (double) floor(8 y * log2(x) + 1)/8 + fmov w2, s0 + scvtf d0, w2, #3 + fmov w2, s3 + scvtf d3, w2, #3 + + // put the 2 bit (0.5) through the roof of twoto1o2mask (make it 0x0 or 0xffffffffffffffff) + uqshl d1, d1, #62 + + // put the 1 bit (0.25) through the roof of twoto1o4mask (make it 0x0 or 0xffffffffffffffff) + uqshl d4, d4, #63 + + // center y*log2(x) fractional part between -0.125 and 0.125 by subtracting (double) floor(8 y * log2(x) + 1)/8 + fsub d17, d17, d0 + // center y*n fractional part between -0.125 and 0.125 by subtracting (double) floor(8 y * n + 1)/8 + fsub d18, d18, d3 + + // Add fractional parts of yn and y log2(x) together + fadd d16, d17, d18 + + // Result = 1.0 (offset for exp(s) series) + fmov d0, #1.0 + + // multiply fractional part of y * log2(x) by ln(2) + fmul d16, d5, d16 + +//@@@ 10th order polynomial series for Exp(s) @@@@ + + // ss2 = (ss)^2 + fmul d17, d16, d16 + + // twoto1o2mask = twoto1o2mask & twoto1o4 + and v1.8B, v1.8B, v6.8B + // twoto1o2mask = twoto1o2mask & twoto1o4 + and v4.8B, v4.8B, v6.8B + + // Result = 1.0 + ss + fadd d0, d0, d16 + + // k7 = ss k8 + k7 +#ifdef NO_FUSED_MULTIPLY + fmul d31, d16, v20.2D[0] + fadd d21, d21, d31 +#else + fmla d21, d16, v20.2D[0] +#endif + // ss4 = (ss*ss) * (ss*ss) + fmul d18, d17, d17 + + // twoto1o2mask = twoto1o2mask | (double) 1.0 - results in either 1.0 or 2^(1/4) in twoto1o2mask + orr v1.8B, v1.8B, v28.8B + // twoto1o2mask = twoto1o4mask | (double) 1.0 - results in either 1.0 or 2^(1/4) in twoto1o4mask + orr v4.8B, v4.8B, v28.8B + + // sign could be set up here, but for now expadjustment = 1.0 + fmov d7, #1.0 + + // ss3 = (ss*ss) * ss + fmul d19, d17, d16 + + // k0 = 1/2 (first non-unity coefficient) + fmov d28, #0.5 + + // Mask out non-exponent bits to make sure we have just 2^int_n + and v2.8B, v2.8B, v30.8B + + // square twoto1o2mask to get 1.0 or 2^(1/2) + fmul d1, d1, d1 + + // multiply twoto2o4mask into the exponent output adjustment value + fmul d7, d7, d4 + +#ifdef NO_FUSED_MULTIPLY + // k5 = ss k6 + k5 + fmul d31, d16, v22.2D[0] + fadd d23, d23, d31 + + // k3 = ss k4 + k3 + fmul d31, d16, v24.2D[0] + fadd d25, d25, d31 + + // k1 = ss k2 + k1 + fmul d31, d16, v26.2D[0] + fadd d27, d27, d31 +#else + // k5 = ss k6 + k5 + fmla d23, d16, v22.2D[0] + + // k3 = ss k4 + k3 + fmla d25, d16, v24.2D[0] + + // k1 = ss k2 + k1 + fmla d27, d16, v26.2D[0] +#endif + // multiply twoto1o2mask into exponent output adjustment value + fmul d7, d7, d1 +#ifdef NO_FUSED_MULTIPLY + // k5 = ss^2 ( ss k8 + k7 ) + ss k6 + k5 + fmul d31, d17, v21.2D[0] + fadd d23, d23, d31 + + // k1 = ss^2 ( ss k4 + k3 ) + ss k2 + k1 + fmul d31, d17, v25.2D[0] + fadd d27, d27, d31 + + // Result = 1.0 + ss + 1/2 ss^2 + fmul d31, d17, v28.2D[0] + fadd d0, d0, d31 +#else + // k5 = ss^2 ( ss k8 + k7 ) + ss k6 + k5 + fmla d23, d17, v21.2D[0] + + // k1 = ss^2 ( ss k4 + k3 ) + ss k2 + k1 + fmla d27, d17, v25.2D[0] + + // Result = 1.0 + ss + 1/2 ss^2 + fmla d0, d17, v28.2D[0] +#endif + // Adjust int_n so that it's a double precision value that can be multiplied by Result + add d7, d2, d7 +#ifdef NO_FUSED_MULTIPLY + // k1 = ss^4 ( ss^2 ( ss k8 + k7 ) + ss k6 + k5 ) + ss^2 ( ss k4 + k3 ) + ss k2 + k1 + fmul d31, d18, v23.2D[0] + fadd d27, d27, d31 + + // Result = 1.0 + ss + 1/2 ss^2 + ss^3 ( ss^4 ( ss^2 ( ss k8 + k7 ) + ss k6 + k5 ) + ss^2 ( ss k4 + k3 ) + ss k2 + k1 ) + fmul d31, d19, v27.2D[0] + fadd d0, d0, d31 +#else + // k1 = ss^4 ( ss^2 ( ss k8 + k7 ) + ss k6 + k5 ) + ss^2 ( ss k4 + k3 ) + ss k2 + k1 + fmla d27, d18, v23.2D[0] + + // Result = 1.0 + ss + 1/2 ss^2 + ss^3 ( ss^4 ( ss^2 ( ss k8 + k7 ) + ss k6 + k5 ) + ss^2 ( ss k4 + k3 ) + ss k2 + k1 ) + fmla d0, d19, v27.2D[0] +#endif + // multiply by adjustment (sign*(rounding ? sqrt(2) : 1) * 2^int_n) + fmul d0, d7, d0 + +.LleavePow: +#if defined(KRAIT_NO_AAPCS_VFP_MODE) + // return Result (FP) + // fmov x0, d0 +#endif +.LleavePowDirect: + // leave directly returning whatever is in d0 + ret +.LuseFullImpl: + fmov d0, x0 + fmov d1, x1 + b __full_ieee754_pow + +.align 6 +.LliteralTable: +// Least-sqares tuned constants for 11th order (log2((1+s)/(1-s)): +.LL4: // ~3/11 + .long 0x53a79915, 0x3fd1b108 +.LL3: // ~1/3 + .long 0x9ca0567a, 0x3fd554fa +.LL2: // ~3/7 + .long 0x1408e660, 0x3fdb6db7 +.LL1: // ~3/5 + .long 0x332D4313, 0x3fe33333 + +// Least-squares tuned constants for 10th order exp(s): +.LE10: // ~1/3628800 + .long 0x25c7ba0a, 0x3e92819b +.LE9: // ~1/362880 + .long 0x9499b49c, 0x3ec72294 +.LE8: // ~1/40320 + .long 0xabb79d95, 0x3efa019f +.LE7: // ~1/5040 + .long 0x8723aeaa, 0x3f2a019f +.LE6: // ~1/720 + .long 0x16c76a94, 0x3f56c16c +.LE5: // ~1/120 + .long 0x11185da8, 0x3f811111 +.LE4: // ~1/24 + .long 0x5555551c, 0x3fa55555 +.LE3: // ~1/6 + .long 0x555554db, 0x3fc55555 + +.LbpA: // (2^(2/5) - 1) + .long 0x4ee54db1, 0x3fd472d1 + +.LbpB: // (2^(4/5) - 2^(2/5)) + .long 0x1c8a36cf, 0x3fdafb62 + +.Ltwofifths: // 2/5 + .long 0x9999999a, 0x3fd99999 + +.Ltwooverthreeln2: + .long 0xDC3A03FD, 0x3FEEC709 + +.Ltwoto1o5: // 2^(1/5) exponent 3ff stripped for non-normalized compares + .long 0x86BAE675, 0x00026111 + +.Ltwoto3o5: // 2^(3/5) exponent 3ff stripped for non-normalized compares + .long 0x03B2AE5C, 0x00084060 + +.Lln2: // ln(2) + .long 0xFEFA39EF, 0x3FE62E42 + +.Ltwoto1o4: // 2^1/4 + .long 0x0a31b715, 0x3ff306fe +END(pow) diff --git a/libm/arm64/floor.S b/libm/arm64/floor.S new file mode 100644 index 000000000..2d792e555 --- /dev/null +++ b/libm/arm64/floor.S @@ -0,0 +1,27 @@ +/* + * Copyright 2015, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +ENTRY(floor) + frintM d0, d0 + ret +END(floor) + +ENTRY(floorf) + frintM s0, s0 + ret +END(floorf) diff --git a/libm/arm64/fma.S b/libm/arm64/fma.S new file mode 100644 index 000000000..e64e8077e --- /dev/null +++ b/libm/arm64/fma.S @@ -0,0 +1,27 @@ +/* + * Copyright 2015, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +ENTRY(fma) + fmadd d0, d0, d1, d2 + ret +END(fma) + +ENTRY(fmaf) + fmadd s0, s0, s1, s2 + ret +END(fmaf) diff --git a/libm/arm64/lrint.S b/libm/arm64/lrint.S new file mode 100644 index 000000000..69cc10c3e --- /dev/null +++ b/libm/arm64/lrint.S @@ -0,0 +1,36 @@ +/* + * Copyright 2015, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +ENTRY(lrint) + frintX d0, d0 + fcvtzs x0, d0 + ret +END(lrint) + +ENTRY(lrintf) + frintX s0, s0 + fcvtzs x0, s0 + ret +END(lrintf) + +// sizeof(long) and sizeof(long long) are the same for aarch64 +.weak llrint +.equ llrint,lrint + +.weak llrintf +.equ llrintf,lrintf diff --git a/libm/arm64/rint.S b/libm/arm64/rint.S new file mode 100644 index 000000000..0820c2211 --- /dev/null +++ b/libm/arm64/rint.S @@ -0,0 +1,27 @@ +/* + * Copyright 2015, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +ENTRY(rint) + frintX d0, d0 + ret +END(rint) + +ENTRY(rintf) + frintX s0, s0 + ret +END(rintf) diff --git a/libm/arm64/sqrt.S b/libm/arm64/sqrt.S new file mode 100644 index 000000000..fe0020bd6 --- /dev/null +++ b/libm/arm64/sqrt.S @@ -0,0 +1,27 @@ +/* + * Copyright 2015, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +ENTRY(sqrt) + fsqrt d0, d0 + ret +END(sqrt) + +ENTRY(sqrtf) + fsqrt s0, s0 + ret +END(sqrtf) diff --git a/libm/arm64/trunc.S b/libm/arm64/trunc.S new file mode 100644 index 000000000..329c08d12 --- /dev/null +++ b/libm/arm64/trunc.S @@ -0,0 +1,27 @@ +/* + * Copyright 2015, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +ENTRY(trunc) + frintZ d0, d0 + ret +END(trunc) + +ENTRY(truncf) + frintZ s0, s0 + ret +END(truncf) diff --git a/libm/include/complex.h b/libm/include/complex.h index 0702541c4..ff6b166f4 100644 --- a/libm/include/complex.h +++ b/libm/include/complex.h @@ -46,14 +46,39 @@ _Static_assert(__generic(_Complex_I, float _Complex, 1, 0), #define complex _Complex #define I _Complex_I +#if __ISO_C_VISIBLE >= 2011 +#ifdef __clang__ +#define CMPLX(x, y) ((double complex){ x, y }) +#define CMPLXF(x, y) ((float complex){ x, y }) +#define CMPLXL(x, y) ((long double complex){ x, y }) +#elif __GNUC_PREREQ__(4, 7) +#define CMPLX(x, y) __builtin_complex((double)(x), (double)(y)) +#define CMPLXF(x, y) __builtin_complex((float)(x), (float)(y)) +#define CMPLXL(x, y) __builtin_complex((long double)(x), (long double)(y)) +#endif +#endif /* __ISO_C_VISIBLE >= 2011 */ + __BEGIN_DECLS +#pragma GCC visibility push(default) double cabs(double complex); float cabsf(float complex); long double cabsl(long double complex); +double complex cacos(double complex); +float complex cacosf(float complex); +double complex cacosh(double complex); +float complex cacoshf(float complex); double carg(double complex); float cargf(float complex); long double cargl(long double complex); +double complex casin(double complex); +float complex casinf(float complex); +double complex casinh(double complex); +float complex casinhf(float complex); +double complex catan(double complex); +float complex catanf(float complex); +double complex catanh(double complex); +float complex catanhf(float complex); double complex ccos(double complex); float complex ccosf(float complex); double complex ccosh(double complex); @@ -87,6 +112,7 @@ float complex ctanf(float complex); double complex ctanh(double complex); float complex ctanhf(float complex); +#pragma GCC visibility pop __END_DECLS #endif /* _COMPLEX_H */ diff --git a/libm/include/math.h b/libm/include/math.h index 3eca1407d..dec6389f6 100644 --- a/libm/include/math.h +++ b/libm/include/math.h @@ -459,9 +459,12 @@ long double tanhl(long double); long double tanl(long double); long double tgammal(long double); long double truncl(long double); - #endif /* __ISO_C_VISIBLE >= 1999 */ +#if __BSD_VISIBLE +long double lgammal_r(long double, int *); +#endif + #if defined(_GNU_SOURCE) void sincos(double, double*, double*); void sincosf(float, float*, float*); diff --git a/libm/upstream-freebsd/lib/msun/ld128/e_lgammal_r.c b/libm/upstream-freebsd/lib/msun/ld128/e_lgammal_r.c new file mode 100644 index 000000000..13f8f75a4 --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/ld128/e_lgammal_r.c @@ -0,0 +1,333 @@ +/* @(#)e_lgamma_r.c 1.3 95/01/18 */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +#include +__FBSDID("$FreeBSD$"); + +/* + * See e_lgamma_r.c for complete comments. + * + * Converted to long double by Steven G. Kargl. + */ + +#include "fpmath.h" +#include "math.h" +#include "math_private.h" + +static const volatile double vzero = 0; + +static const double +zero= 0, +half= 0.5, +one = 1; + +static const long double +pi = 3.14159265358979323846264338327950288e+00L; +/* + * Domain y in [0x1p-119, 0.28], range ~[-1.4065e-36, 1.4065e-36]: + * |(lgamma(2 - y) + y / 2) / y - a(y)| < 2**-119.1 + */ +static const long double +a0 = 7.72156649015328606065120900824024296e-02L, +a1 = 3.22467033424113218236207583323018498e-01L, +a2 = 6.73523010531980951332460538330282217e-02L, +a3 = 2.05808084277845478790009252803463129e-02L, +a4 = 7.38555102867398526627292839296001626e-03L, +a5 = 2.89051033074152328576829509522483468e-03L, +a6 = 1.19275391170326097618357349881842913e-03L, +a7 = 5.09669524743042462515256340206203019e-04L, +a8 = 2.23154758453578096143609255559576017e-04L, +a9 = 9.94575127818397632126978731542755129e-05L, +a10 = 4.49262367375420471287545895027098145e-05L, +a11 = 2.05072127845117995426519671481628849e-05L, +a12 = 9.43948816959096748454087141447939513e-06L, +a13 = 4.37486780697359330303852050718287419e-06L, +a14 = 2.03920783892362558276037363847651809e-06L, +a15 = 9.55191070057967287877923073200324649e-07L, +a16 = 4.48993286185740853170657139487620560e-07L, +a17 = 2.13107543597620911675316728179563522e-07L, +a18 = 9.70745379855304499867546549551023473e-08L, +a19 = 5.61889970390290257926487734695402075e-08L, +a20 = 6.42739653024130071866684358960960951e-09L, +a21 = 3.34491062143649291746195612991870119e-08L, +a22 = -1.57068547394315223934653011440641472e-08L, +a23 = 1.30812825422415841213733487745200632e-08L; +/* + * Domain x in [tc-0.24, tc+0.28], range ~[-6.3201e-37, 6.3201e-37]: + * |(lgamma(x) - tf) - t(x - tc)| < 2**-120.3. + */ +static const long double +tc = 1.46163214496836234126265954232572133e+00L, +tf = -1.21486290535849608095514557177691584e-01L, +tt = 1.57061739945077675484237837992951704e-36L, +t0 = -1.99238329499314692728655623767019240e-36L, +t1 = -6.08453430711711404116887457663281416e-35L, +t2 = 4.83836122723810585213722380854828904e-01L, +t3 = -1.47587722994530702030955093950668275e-01L, +t4 = 6.46249402389127526561003464202671923e-02L, +t5 = -3.27885410884813055008502586863748063e-02L, +t6 = 1.79706751152103942928638276067164935e-02L, +t7 = -1.03142230366363872751602029672767978e-02L, +t8 = 6.10053602051788840313573150785080958e-03L, +t9 = -3.68456960831637325470641021892968954e-03L, +t10 = 2.25976482322181046611440855340968560e-03L, +t11 = -1.40225144590445082933490395950664961e-03L, +t12 = 8.78232634717681264035014878172485575e-04L, +t13 = -5.54194952796682301220684760591403899e-04L, +t14 = 3.51912956837848209220421213975000298e-04L, +t15 = -2.24653443695947456542669289367055542e-04L, +t16 = 1.44070395420840737695611929680511823e-04L, +t17 = -9.27609865550394140067059487518862512e-05L, +t18 = 5.99347334438437081412945428365433073e-05L, +t19 = -3.88458388854572825603964274134801009e-05L, +t20 = 2.52476631610328129217896436186551043e-05L, +t21 = -1.64508584981658692556994212457518536e-05L, +t22 = 1.07434583475987007495523340296173839e-05L, +t23 = -7.03070407519397260929482550448878399e-06L, +t24 = 4.60968590693753579648385629003100469e-06L, +t25 = -3.02765473778832036018438676945512661e-06L, +t26 = 1.99238771545503819972741288511303401e-06L, +t27 = -1.31281299822614084861868817951788579e-06L, +t28 = 8.60844432267399655055574642052370223e-07L, +t29 = -5.64535486432397413273248363550536374e-07L, +t30 = 3.99357783676275660934903139592727737e-07L, +t31 = -2.95849029193433121795495215869311610e-07L, +t32 = 1.37790144435073124976696250804940384e-07L; +/* + * Domain y in [-0.1, 0.232], range ~[-1.4046e-37, 1.4181e-37]: + * |(lgamma(1 + y) + 0.5 * y) / y - u(y) / v(y)| < 2**-122.8 + */ +static const long double +u0 = -7.72156649015328606065120900824024311e-02L, +u1 = 4.24082772271938167430983113242482656e-01L, +u2 = 2.96194003481457101058321977413332171e+00L, +u3 = 6.49503267711258043997790983071543710e+00L, +u4 = 7.40090051288150177152835698948644483e+00L, +u5 = 4.94698036296756044610805900340723464e+00L, +u6 = 2.00194224610796294762469550684947768e+00L, +u7 = 4.82073087750608895996915051568834949e-01L, +u8 = 6.46694052280506568192333848437585427e-02L, +u9 = 4.17685526755100259316625348933108810e-03L, +u10 = 9.06361003550314327144119307810053410e-05L, +v1 = 5.15937098592887275994320496999951947e+00L, +v2 = 1.14068418766251486777604403304717558e+01L, +v3 = 1.41164839437524744055723871839748489e+01L, +v4 = 1.07170702656179582805791063277960532e+01L, +v5 = 5.14448694179047879915042998453632434e+00L, +v6 = 1.55210088094585540637493826431170289e+00L, +v7 = 2.82975732849424562719893657416365673e-01L, +v8 = 2.86424622754753198010525786005443539e-02L, +v9 = 1.35364253570403771005922441442688978e-03L, +v10 = 1.91514173702398375346658943749580666e-05L, +v11 = -3.25364686890242327944584691466034268e-08L; +/* + * Domain x in (2, 3], range ~[-1.3341e-36, 1.3536e-36]: + * |(lgamma(y+2) - 0.5 * y) / y - s(y)/r(y)| < 2**-120.1 + * with y = x - 2. + */ +static const long double +s0 = -7.72156649015328606065120900824024297e-02L, +s1 = 1.23221687850916448903914170805852253e-01L, +s2 = 5.43673188699937239808255378293820020e-01L, +s3 = 6.31998137119005233383666791176301800e-01L, +s4 = 3.75885340179479850993811501596213763e-01L, +s5 = 1.31572908743275052623410195011261575e-01L, +s6 = 2.82528453299138685507186287149699749e-02L, +s7 = 3.70262021550340817867688714880797019e-03L, +s8 = 2.83374000312371199625774129290973648e-04L, +s9 = 1.15091830239148290758883505582343691e-05L, +s10 = 2.04203474281493971326506384646692446e-07L, +s11 = 9.79544198078992058548607407635645763e-10L, +r1 = 2.58037466655605285937112832039537492e+00L, +r2 = 2.86289413392776399262513849911531180e+00L, +r3 = 1.78691044735267497452847829579514367e+00L, +r4 = 6.89400381446725342846854215600008055e-01L, +r5 = 1.70135865462567955867134197595365343e-01L, +r6 = 2.68794816183964420375498986152766763e-02L, +r7 = 2.64617234244861832870088893332006679e-03L, +r8 = 1.52881761239180800640068128681725702e-04L, +r9 = 4.63264813762296029824851351257638558e-06L, +r10 = 5.89461519146957343083848967333671142e-08L, +r11 = 1.79027678176582527798327441636552968e-10L; +/* + * Domain z in [8, 0x1p70], range ~[-9.8214e-35, 9.8214e-35]: + * |lgamma(x) - (x - 0.5) * (log(x) - 1) - w(1/x)| < 2**-113.0 + */ +static const long double +w0 = 4.18938533204672741780329736405617738e-01L, +w1 = 8.33333333333333333333333333332852026e-02L, +w2 = -2.77777777777777777777777727810123528e-03L, +w3 = 7.93650793650793650791708939493907380e-04L, +w4 = -5.95238095238095234390450004444370959e-04L, +w5 = 8.41750841750837633887817658848845695e-04L, +w6 = -1.91752691752396849943172337347259743e-03L, +w7 = 6.41025640880333069429106541459015557e-03L, +w8 = -2.95506530801732133437990433080327074e-02L, +w9 = 1.79644237328444101596766586979576927e-01L, +w10 = -1.39240539108367641920172649259736394e+00L, +w11 = 1.33987701479007233325288857758641761e+01L, +w12 = -1.56363596431084279780966590116006255e+02L, +w13 = 2.14830978044410267201172332952040777e+03L, +w14 = -3.28636067474227378352761516589092334e+04L, +w15 = 5.06201257747865138432663574251462485e+05L, +w16 = -6.79720123352023636706247599728048344e+06L, +w17 = 6.57556601705472106989497289465949255e+07L, +w18 = -3.26229058141181783534257632389415580e+08L; + +static long double +sin_pil(long double x) +{ + volatile long double vz; + long double y,z; + uint64_t lx, n; + uint16_t hx; + + y = -x; + + vz = y+0x1.p112; + z = vz-0x1.p112; + if (z == y) + return zero; + + vz = y+0x1.p110; + EXTRACT_LDBL128_WORDS(hx,lx,n,vz); + z = vz-0x1.p110; + if (z > y) { + z -= 0.25; + n--; + } + n &= 7; + y = y - z + n * 0.25L; + + switch (n) { + case 0: y = __kernel_sinl(pi*y,zero,0); break; + case 1: + case 2: y = __kernel_cosl(pi*(0.5-y),zero); break; + case 3: + case 4: y = __kernel_sinl(pi*(one-y),zero,0); break; + case 5: + case 6: y = -__kernel_cosl(pi*(y-1.5),zero); break; + default: y = __kernel_sinl(pi*(y-2.0),zero,0); break; + } + return -y; +} + + +long double +lgammal_r(long double x, int *signgamp) +{ + long double nadj,p,p1,p2,p3,q,r,t,w,y,z; + uint64_t llx,lx; + int i; + uint16_t hx; + + EXTRACT_LDBL128_WORDS(hx, lx, llx, x); + + if((hx & 0x7fff) == 0x7fff) { /* erfl(nan)=nan */ + i = (hx>>15)<<1; + return (1-i)+one/x; /* erfl(+-inf)=+-1 */ + } + + /* purge off +-inf, NaN, +-0, tiny and negative arguments */ + *signgamp = 1; + if((hx & 0x7fff) == 0x7fff) /* x is +-Inf or NaN */ + return x*x; + if((hx==0||hx==0x8000)&&lx==0) { + if (hx&0x8000) + *signgamp = -1; + return one/vzero; + } + + /* purge off tiny and negative arguments */ + if(fabsl(x)<0x1p-119L) { + if(hx&0x8000) { + *signgamp = -1; + return -logl(-x); + } else return -logl(x); + } + if(hx&0x8000) { + if(fabsl(x)>=0x1p112) + return one/vzero; + t = sin_pil(x); + if(t==zero) return one/vzero; + nadj = logl(pi/fabsl(t*x)); + if(t=0.7315998077392578) {y = 1-x; i= 0;} + else if(x>=0.2316399812698364) {y= x-(tc-1); i=1;} + else {y = x; i=2;} + } else { + r = 0; + if(x>=1.7316312789916992) {y=2-x;i=0;} + else if(x>=1.2316322326660156) {y=x-tc;i=1;} + else {y=x-1;i=2;} + } + switch(i) { + case 0: + z = y*y; + p1 = a0+z*(a2+z*(a4+z*(a6+z*(a8+z*(a10+z*(a12+z*(a14+z*(a16+ + z*(a18+z*(a20+z*a22)))))))))); + p2 = z*(a1+z*(a3+z*(a5+z*(a7+z*(a9+z*(a11+z*(a13+z*(a15+ + z*(a17+z*(a19+z*(a21+z*a23))))))))))); + p = y*p1+p2; + r += (p-y/2); break; + case 1: + p = t0+y*t1+tt+y*y*(t2+y*(t3+y*(t4+y*(t5+y*(t6+y*(t7+y*(t8+ + y*(t9+y*(t10+y*(t11+y*(t12+y*(t13+y*(t14+y*(t15+y*(t16+ + y*(t17+y*(t18+y*(t19+y*(t20+y*(t21+y*(t22+y*(t23+ + y*(t24+y*(t25+y*(t26+y*(t27+y*(t28+y*(t29+y*(t30+ + y*(t31+y*t32)))))))))))))))))))))))))))))); + r += (tf + p); break; + case 2: + p1 = y*(u0+y*(u1+y*(u2+y*(u3+y*(u4+y*(u5+y*(u6+y*(u7+ + y*(u8+y*(u9+y*u10)))))))))); + p2 = one+y*(v1+y*(v2+y*(v3+y*(v4+y*(v5+y*(v6+y*(v7+ + y*(v8+y*(v9+y*(v10+y*v11)))))))))); + r += (-y/2 + p1/p2); + } + } + else if(x<8) { + i = x; + y = x-i; + p = y*(s0+y*(s1+y*(s2+y*(s3+y*(s4+y*(s5+y*(s6+y*(s7+y*(s8+ + y*(s9+y*(s10+y*s11))))))))))); + q = one+y*(r1+y*(r2+y*(r3+y*(r4+y*(r5+y*(r6+y*(r7+y*(r8+ + y*(r9+y*(r10+y*r11)))))))))); + r = y/2+p/q; + z = 1; /* lgamma(1+s) = log(s) + lgamma(s) */ + switch(i) { + case 7: z *= (y+6); /* FALLTHRU */ + case 6: z *= (y+5); /* FALLTHRU */ + case 5: z *= (y+4); /* FALLTHRU */ + case 4: z *= (y+3); /* FALLTHRU */ + case 3: z *= (y+2); /* FALLTHRU */ + r += logl(z); break; + } + } else if (x < 0x1p119L) { + t = logl(x); + z = one/x; + y = z*z; + w = w0+z*(w1+y*(w2+y*(w3+y*(w4+y*(w5+y*(w6+y*(w7+y*(w8+ + y*(w9+y*(w10+y*(w11+y*(w12+y*(w13+y*(w14+y*(w15+y*(w16+ + y*(w17+y*w18))))))))))))))))); + r = (x-half)*(t-one)+w; + } else + r = x*(logl(x)-1); + if(hx&0x8000) r = nadj - r; + return r; +} diff --git a/libm/upstream-freebsd/lib/msun/ld128/e_rem_pio2l.h b/libm/upstream-freebsd/lib/msun/ld128/e_rem_pio2l.h index 078d0c3d4..5d78c4de0 100644 --- a/libm/upstream-freebsd/lib/msun/ld128/e_rem_pio2l.h +++ b/libm/upstream-freebsd/lib/msun/ld128/e_rem_pio2l.h @@ -6,7 +6,7 @@ * * Developed at SunSoft, a Sun Microsystems, Inc. business. * Permission to use, copy, modify, and distribute this - * software is freely granted, provided that this notice + * software is freely granted, provided that this notice * is preserved. * ==================================================== * @@ -17,8 +17,8 @@ __FBSDID("$FreeBSD$"); /* ld128 version of __ieee754_rem_pio2l(x,y) - * - * return the remainder of x rem pi/2 in y[0]+y[1] + * + * return the remainder of x rem pi/2 in y[0]+y[1] * use __kernel_rem_pio2() */ @@ -88,24 +88,24 @@ __ieee754_rem_pio2l(long double x, long double *y) union IEEEl2bits u2; int ex1; j = ex; - y[0] = r-w; + y[0] = r-w; u2.e = y[0]; ex1 = u2.xbits.expsign & 0x7fff; i = j-ex1; if(i>51) { /* 2nd iteration needed, good to 248 */ t = r; - w = fn*pio2_2; + w = fn*pio2_2; r = t-w; - w = fn*pio2_2t-((t-r)-w); + w = fn*pio2_2t-((t-r)-w); y[0] = r-w; u2.e = y[0]; ex1 = u2.xbits.expsign & 0x7fff; i = j-ex1; if(i>119) { /* 3rd iteration need, 316 bits acc */ t = r; /* will cover all possible cases */ - w = fn*pio2_3; + w = fn*pio2_3; r = t-w; - w = fn*pio2_3t-((t-r)-w); + w = fn*pio2_3t-((t-r)-w); y[0] = r-w; } } @@ -113,7 +113,7 @@ __ieee754_rem_pio2l(long double x, long double *y) y[1] = (r-y[0])-w; return n; } - /* + /* * all other (large) arguments */ if(ex==0x7fff) { /* x is inf or NaN */ diff --git a/libm/upstream-freebsd/lib/msun/ld128/invtrig.c b/libm/upstream-freebsd/lib/msun/ld128/invtrig.c index df67d16f5..4ceca8a45 100644 --- a/libm/upstream-freebsd/lib/msun/ld128/invtrig.c +++ b/libm/upstream-freebsd/lib/msun/ld128/invtrig.c @@ -58,8 +58,8 @@ qS9 = -1.99407384882605586705979504567947007e-04L; */ const long double atanhi[] = { 4.63647609000806116214256231461214397e-01L, - 7.85398163397448309615660845819875699e-01L, - 9.82793723247329067985710611014666038e-01L, + 7.85398163397448309615660845819875699e-01L, + 9.82793723247329067985710611014666038e-01L, 1.57079632679489661923132169163975140e+00L, }; diff --git a/libm/upstream-freebsd/lib/msun/ld128/k_cosl.c b/libm/upstream-freebsd/lib/msun/ld128/k_cosl.c index 5f4aa378d..fe57773a1 100644 --- a/libm/upstream-freebsd/lib/msun/ld128/k_cosl.c +++ b/libm/upstream-freebsd/lib/msun/ld128/k_cosl.c @@ -6,7 +6,7 @@ * * Developed at SunSoft, a Sun Microsystems, Inc. business. * Permission to use, copy, modify, and distribute this - * software is freely granted, provided that this notice + * software is freely granted, provided that this notice * is preserved. * ==================================================== */ diff --git a/libm/upstream-freebsd/lib/msun/ld128/k_expl.h b/libm/upstream-freebsd/lib/msun/ld128/k_expl.h new file mode 100644 index 000000000..a5668fd47 --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/ld128/k_expl.h @@ -0,0 +1,328 @@ +/* from: FreeBSD: head/lib/msun/ld128/s_expl.c 251345 2013-06-03 20:09:22Z kargl */ + +/*- + * Copyright (c) 2009-2013 Steven G. Kargl + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Optimized by Bruce D. Evans. + */ + +#include +__FBSDID("$FreeBSD$"); + +/* + * ld128 version of k_expl.h. See ../ld80/s_expl.c for most comments. + * + * See ../src/e_exp.c and ../src/k_exp.h for precision-independent comments + * about the secondary kernels. + */ + +#define INTERVALS 128 +#define LOG2_INTERVALS 7 +#define BIAS (LDBL_MAX_EXP - 1) + +static const double +/* + * ln2/INTERVALS = L1+L2 (hi+lo decomposition for multiplication). L1 must + * have at least 22 (= log2(|LDBL_MIN_EXP-extras|) + log2(INTERVALS)) lowest + * bits zero so that multiplication of it by n is exact. + */ +INV_L = 1.8466496523378731e+2, /* 0x171547652b82fe.0p-45 */ +L2 = -1.0253670638894731e-29; /* -0x1.9ff0342542fc3p-97 */ +static const long double +/* 0x1.62e42fefa39ef35793c768000000p-8 */ +L1 = 5.41521234812457272982212595914567508e-3L; + +/* + * XXX values in hex in comments have been lost (or were never present) + * from here. + */ +static const long double +/* + * Domain [-0.002708, 0.002708], range ~[-2.4021e-38, 2.4234e-38]: + * |exp(x) - p(x)| < 2**-124.9 + * (0.002708 is ln2/(2*INTERVALS) rounded up a little). + * + * XXX the coeffs aren't very carefully rounded, and I get 3.6 more bits. + */ +A2 = 0.5, +A3 = 1.66666666666666666666666666651085500e-1L, +A4 = 4.16666666666666666666666666425885320e-2L, +A5 = 8.33333333333333333334522877160175842e-3L, +A6 = 1.38888888888888888889971139751596836e-3L; + +static const double +A7 = 1.9841269841269470e-4, /* 0x1.a01a01a019f91p-13 */ +A8 = 2.4801587301585286e-5, /* 0x1.71de3ec75a967p-19 */ +A9 = 2.7557324277411235e-6, /* 0x1.71de3ec75a967p-19 */ +A10 = 2.7557333722375069e-7; /* 0x1.27e505ab56259p-22 */ + +static const struct { + /* + * hi must be rounded to at most 106 bits so that multiplication + * by r1 in expm1l() is exact, but it is rounded to 88 bits due to + * historical accidents. + * + * XXX it is wasteful to use long double for both hi and lo. ld128 + * exp2l() uses only float for lo (in a very differently organized + * table; ld80 exp2l() is different again. It uses 2 doubles in a + * table organized like this one. 1 double and 1 float would + * suffice). There are different packing/locality/alignment/caching + * problems with these methods. + * + * XXX C's bad %a format makes the bits unreadable. They happen + * to all line up for the hi values 1 before the point and 88 + * in 22 nybbles, but for the low values the nybbles are shifted + * randomly. + */ + long double hi; + long double lo; +} tbl[INTERVALS] = { + 0x1p0L, 0x0p0L, + 0x1.0163da9fb33356d84a66aep0L, 0x3.36dcdfa4003ec04c360be2404078p-92L, + 0x1.02c9a3e778060ee6f7cacap0L, 0x4.f7a29bde93d70a2cabc5cb89ba10p-92L, + 0x1.04315e86e7f84bd738f9a2p0L, 0xd.a47e6ed040bb4bfc05af6455e9b8p-96L, + 0x1.059b0d31585743ae7c548ep0L, 0xb.68ca417fe53e3495f7df4baf84a0p-92L, + 0x1.0706b29ddf6ddc6dc403a8p0L, 0x1.d87b27ed07cb8b092ac75e311753p-88L, + 0x1.0874518759bc808c35f25cp0L, 0x1.9427fa2b041b2d6829d8993a0d01p-88L, + 0x1.09e3ecac6f3834521e060cp0L, 0x5.84d6b74ba2e023da730e7fccb758p-92L, + 0x1.0b5586cf9890f6298b92b6p0L, 0x1.1842a98364291408b3ceb0a2a2bbp-88L, + 0x1.0cc922b7247f7407b705b8p0L, 0x9.3dc5e8aac564e6fe2ef1d431fd98p-92L, + 0x1.0e3ec32d3d1a2020742e4ep0L, 0x1.8af6a552ac4b358b1129e9f966a4p-88L, + 0x1.0fb66affed31af232091dcp0L, 0x1.8a1426514e0b627bda694a400a27p-88L, + 0x1.11301d0125b50a4ebbf1aep0L, 0xd.9318ceac5cc47ab166ee57427178p-92L, + 0x1.12abdc06c31cbfb92bad32p0L, 0x4.d68e2f7270bdf7cedf94eb1cb818p-92L, + 0x1.1429aaea92ddfb34101942p0L, 0x1.b2586d01844b389bea7aedd221d4p-88L, + 0x1.15a98c8a58e512480d573cp0L, 0x1.d5613bf92a2b618ee31b376c2689p-88L, + 0x1.172b83c7d517adcdf7c8c4p0L, 0x1.0eb14a792035509ff7d758693f24p-88L, + 0x1.18af9388c8de9bbbf70b9ap0L, 0x3.c2505c97c0102e5f1211941d2840p-92L, + 0x1.1a35beb6fcb753cb698f68p0L, 0x1.2d1c835a6c30724d5cfae31b84e5p-88L, + 0x1.1bbe084045cd39ab1e72b4p0L, 0x4.27e35f9acb57e473915519a1b448p-92L, + 0x1.1d4873168b9aa7805b8028p0L, 0x9.90f07a98b42206e46166cf051d70p-92L, + 0x1.1ed5022fcd91cb8819ff60p0L, 0x1.121d1e504d36c47474c9b7de6067p-88L, + 0x1.2063b88628cd63b8eeb028p0L, 0x1.50929d0fc487d21c2b84004264dep-88L, + 0x1.21f49917ddc962552fd292p0L, 0x9.4bdb4b61ea62477caa1dce823ba0p-92L, + 0x1.2387a6e75623866c1fadb0p0L, 0x1.c15cb593b0328566902df69e4de2p-88L, + 0x1.251ce4fb2a63f3582ab7dep0L, 0x9.e94811a9c8afdcf796934bc652d0p-92L, + 0x1.26b4565e27cdd257a67328p0L, 0x1.d3b249dce4e9186ddd5ff44e6b08p-92L, + 0x1.284dfe1f5638096cf15cf0p0L, 0x3.ca0967fdaa2e52d7c8106f2e262cp-92L, + 0x1.29e9df51fdee12c25d15f4p0L, 0x1.a24aa3bca890ac08d203fed80a07p-88L, + 0x1.2b87fd0dad98ffddea4652p0L, 0x1.8fcab88442fdc3cb6de4519165edp-88L, + 0x1.2d285a6e4030b40091d536p0L, 0xd.075384589c1cd1b3e4018a6b1348p-92L, + 0x1.2ecafa93e2f5611ca0f45cp0L, 0x1.523833af611bdcda253c554cf278p-88L, + 0x1.306fe0a31b7152de8d5a46p0L, 0x3.05c85edecbc27343629f502f1af2p-92L, + 0x1.32170fc4cd8313539cf1c2p0L, 0x1.008f86dde3220ae17a005b6412bep-88L, + 0x1.33c08b26416ff4c9c8610cp0L, 0x1.96696bf95d1593039539d94d662bp-88L, + 0x1.356c55f929ff0c94623476p0L, 0x3.73af38d6d8d6f9506c9bbc93cbc0p-92L, + 0x1.371a7373aa9caa7145502ep0L, 0x1.4547987e3e12516bf9c699be432fp-88L, + 0x1.38cae6d05d86585a9cb0d8p0L, 0x1.bed0c853bd30a02790931eb2e8f0p-88L, + 0x1.3a7db34e59ff6ea1bc9298p0L, 0x1.e0a1d336163fe2f852ceeb134067p-88L, + 0x1.3c32dc313a8e484001f228p0L, 0xb.58f3775e06ab66353001fae9fca0p-92L, + 0x1.3dea64c12342235b41223ep0L, 0x1.3d773fba2cb82b8244267c54443fp-92L, + 0x1.3fa4504ac801ba0bf701aap0L, 0x4.1832fb8c1c8dbdff2c49909e6c60p-92L, + 0x1.4160a21f72e29f84325b8ep0L, 0x1.3db61fb352f0540e6ba05634413ep-88L, + 0x1.431f5d950a896dc7044394p0L, 0x1.0ccec81e24b0caff7581ef4127f7p-92L, + 0x1.44e086061892d03136f408p0L, 0x1.df019fbd4f3b48709b78591d5cb5p-88L, + 0x1.46a41ed1d005772512f458p0L, 0x1.229d97df404ff21f39c1b594d3a8p-88L, + 0x1.486a2b5c13cd013c1a3b68p0L, 0x1.062f03c3dd75ce8757f780e6ec99p-88L, + 0x1.4a32af0d7d3de672d8bcf4p0L, 0x6.f9586461db1d878b1d148bd3ccb8p-92L, + 0x1.4bfdad5362a271d4397afep0L, 0xc.42e20e0363ba2e159c579f82e4b0p-92L, + 0x1.4dcb299fddd0d63b36ef1ap0L, 0x9.e0cc484b25a5566d0bd5f58ad238p-92L, + 0x1.4f9b2769d2ca6ad33d8b68p0L, 0x1.aa073ee55e028497a329a7333dbap-88L, + 0x1.516daa2cf6641c112f52c8p0L, 0x4.d822190e718226177d7608d20038p-92L, + 0x1.5342b569d4f81df0a83c48p0L, 0x1.d86a63f4e672a3e429805b049465p-88L, + 0x1.551a4ca5d920ec52ec6202p0L, 0x4.34ca672645dc6c124d6619a87574p-92L, + 0x1.56f4736b527da66ecb0046p0L, 0x1.64eb3c00f2f5ab3d801d7cc7272dp-88L, + 0x1.58d12d497c7fd252bc2b72p0L, 0x1.43bcf2ec936a970d9cc266f0072fp-88L, + 0x1.5ab07dd48542958c930150p0L, 0x1.91eb345d88d7c81280e069fbdb63p-88L, + 0x1.5c9268a5946b701c4b1b80p0L, 0x1.6986a203d84e6a4a92f179e71889p-88L, + 0x1.5e76f15ad21486e9be4c20p0L, 0x3.99766a06548a05829e853bdb2b52p-92L, + 0x1.605e1b976dc08b076f592ap0L, 0x4.86e3b34ead1b4769df867b9c89ccp-92L, + 0x1.6247eb03a5584b1f0fa06ep0L, 0x1.d2da42bb1ceaf9f732275b8aef30p-88L, + 0x1.6434634ccc31fc76f8714cp0L, 0x4.ed9a4e41000307103a18cf7a6e08p-92L, + 0x1.66238825522249127d9e28p0L, 0x1.b8f314a337f4dc0a3adf1787ff74p-88L, + 0x1.68155d44ca973081c57226p0L, 0x1.b9f32706bfe4e627d809a85dcc66p-88L, + 0x1.6a09e667f3bcc908b2fb12p0L, 0x1.66ea957d3e3adec17512775099dap-88L, + 0x1.6c012750bdabeed76a9980p0L, 0xf.4f33fdeb8b0ecd831106f57b3d00p-96L, + 0x1.6dfb23c651a2ef220e2cbep0L, 0x1.bbaa834b3f11577ceefbe6c1c411p-92L, + 0x1.6ff7df9519483cf87e1b4ep0L, 0x1.3e213bff9b702d5aa477c12523cep-88L, + 0x1.71f75e8ec5f73dd2370f2ep0L, 0xf.0acd6cb434b562d9e8a20adda648p-92L, + 0x1.73f9a48a58173bd5c9a4e6p0L, 0x8.ab1182ae217f3a7681759553e840p-92L, + 0x1.75feb564267c8bf6e9aa32p0L, 0x1.a48b27071805e61a17b954a2dad8p-88L, + 0x1.780694fde5d3f619ae0280p0L, 0x8.58b2bb2bdcf86cd08e35fb04c0f0p-92L, + 0x1.7a11473eb0186d7d51023ep0L, 0x1.6cda1f5ef42b66977960531e821bp-88L, + 0x1.7c1ed0130c1327c4933444p0L, 0x1.937562b2dc933d44fc828efd4c9cp-88L, + 0x1.7e2f336cf4e62105d02ba0p0L, 0x1.5797e170a1427f8fcdf5f3906108p-88L, + 0x1.80427543e1a11b60de6764p0L, 0x9.a354ea706b8e4d8b718a672bf7c8p-92L, + 0x1.82589994cce128acf88afap0L, 0xb.34a010f6ad65cbbac0f532d39be0p-92L, + 0x1.8471a4623c7acce52f6b96p0L, 0x1.c64095370f51f48817914dd78665p-88L, + 0x1.868d99b4492ec80e41d90ap0L, 0xc.251707484d73f136fb5779656b70p-92L, + 0x1.88ac7d98a669966530bcdep0L, 0x1.2d4e9d61283ef385de170ab20f96p-88L, + 0x1.8ace5422aa0db5ba7c55a0p0L, 0x1.92c9bb3e6ed61f2733304a346d8fp-88L, + 0x1.8cf3216b5448bef2aa1cd0p0L, 0x1.61c55d84a9848f8c453b3ca8c946p-88L, + 0x1.8f1ae991577362b982745cp0L, 0x7.2ed804efc9b4ae1458ae946099d4p-92L, + 0x1.9145b0b91ffc588a61b468p0L, 0x1.f6b70e01c2a90229a4c4309ea719p-88L, + 0x1.93737b0cdc5e4f4501c3f2p0L, 0x5.40a22d2fc4af581b63e8326efe9cp-92L, + 0x1.95a44cbc8520ee9b483694p0L, 0x1.a0fc6f7c7d61b2b3a22a0eab2cadp-88L, + 0x1.97d829fde4e4f8b9e920f8p0L, 0x1.1e8bd7edb9d7144b6f6818084cc7p-88L, + 0x1.9a0f170ca07b9ba3109b8cp0L, 0x4.6737beb19e1eada6825d3c557428p-92L, + 0x1.9c49182a3f0901c7c46b06p0L, 0x1.1f2be58ddade50c217186c90b457p-88L, + 0x1.9e86319e323231824ca78ep0L, 0x6.4c6e010f92c082bbadfaf605cfd4p-92L, + 0x1.a0c667b5de564b29ada8b8p0L, 0xc.ab349aa0422a8da7d4512edac548p-92L, + 0x1.a309bec4a2d3358c171f76p0L, 0x1.0daad547fa22c26d168ea762d854p-88L, + 0x1.a5503b23e255c8b424491cp0L, 0xa.f87bc8050a405381703ef7caff50p-92L, + 0x1.a799e1330b3586f2dfb2b0p0L, 0x1.58f1a98796ce8908ae852236ca94p-88L, + 0x1.a9e6b5579fdbf43eb243bcp0L, 0x1.ff4c4c58b571cf465caf07b4b9f5p-88L, + 0x1.ac36bbfd3f379c0db966a2p0L, 0x1.1265fc73e480712d20f8597a8e7bp-88L, + 0x1.ae89f995ad3ad5e8734d16p0L, 0x1.73205a7fbc3ae675ea440b162d6cp-88L, + 0x1.b0e07298db66590842acdep0L, 0x1.c6f6ca0e5dcae2aafffa7a0554cbp-88L, + 0x1.b33a2b84f15faf6bfd0e7ap0L, 0x1.d947c2575781dbb49b1237c87b6ep-88L, + 0x1.b59728de559398e3881110p0L, 0x1.64873c7171fefc410416be0a6525p-88L, + 0x1.b7f76f2fb5e46eaa7b081ap0L, 0xb.53c5354c8903c356e4b625aacc28p-92L, + 0x1.ba5b030a10649840cb3c6ap0L, 0xf.5b47f297203757e1cc6eadc8bad0p-92L, + 0x1.bcc1e904bc1d2247ba0f44p0L, 0x1.b3d08cd0b20287092bd59be4ad98p-88L, + 0x1.bf2c25bd71e088408d7024p0L, 0x1.18e3449fa073b356766dfb568ff4p-88L, + 0x1.c199bdd85529c2220cb12ap0L, 0x9.1ba6679444964a36661240043970p-96L, + 0x1.c40ab5fffd07a6d14df820p0L, 0xf.1828a5366fd387a7bdd54cdf7300p-92L, + 0x1.c67f12e57d14b4a2137fd2p0L, 0xf.2b301dd9e6b151a6d1f9d5d5f520p-96L, + 0x1.c8f6d9406e7b511acbc488p0L, 0x5.c442ddb55820171f319d9e5076a8p-96L, + 0x1.cb720dcef90691503cbd1ep0L, 0x9.49db761d9559ac0cb6dd3ed599e0p-92L, + 0x1.cdf0b555dc3f9c44f8958ep0L, 0x1.ac51be515f8c58bdfb6f5740a3a4p-88L, + 0x1.d072d4a07897b8d0f22f20p0L, 0x1.a158e18fbbfc625f09f4cca40874p-88L, + 0x1.d2f87080d89f18ade12398p0L, 0x9.ea2025b4c56553f5cdee4c924728p-92L, + 0x1.d5818dcfba48725da05aeap0L, 0x1.66e0dca9f589f559c0876ff23830p-88L, + 0x1.d80e316c98397bb84f9d04p0L, 0x8.805f84bec614de269900ddf98d28p-92L, + 0x1.da9e603db3285708c01a5ap0L, 0x1.6d4c97f6246f0ec614ec95c99392p-88L, + 0x1.dd321f301b4604b695de3cp0L, 0x6.30a393215299e30d4fb73503c348p-96L, + 0x1.dfc97337b9b5eb968cac38p0L, 0x1.ed291b7225a944efd5bb5524b927p-88L, + 0x1.e264614f5a128a12761fa0p0L, 0x1.7ada6467e77f73bf65e04c95e29dp-88L, + 0x1.e502ee78b3ff6273d13014p0L, 0x1.3991e8f49659e1693be17ae1d2f9p-88L, + 0x1.e7a51fbc74c834b548b282p0L, 0x1.23786758a84f4956354634a416cep-88L, + 0x1.ea4afa2a490d9858f73a18p0L, 0xf.5db301f86dea20610ceee13eb7b8p-92L, + 0x1.ecf482d8e67f08db0312fap0L, 0x1.949cef462010bb4bc4ce72a900dfp-88L, + 0x1.efa1bee615a27771fd21a8p0L, 0x1.2dac1f6dd5d229ff68e46f27e3dfp-88L, + 0x1.f252b376bba974e8696fc2p0L, 0x1.6390d4c6ad5476b5162f40e1d9a9p-88L, + 0x1.f50765b6e4540674f84b76p0L, 0x2.862baff99000dfc4352ba29b8908p-92L, + 0x1.f7bfdad9cbe138913b4bfep0L, 0x7.2bd95c5ce7280fa4d2344a3f5618p-92L, + 0x1.fa7c1819e90d82e90a7e74p0L, 0xb.263c1dc060c36f7650b4c0f233a8p-92L, + 0x1.fd3c22b8f71f10975ba4b2p0L, 0x1.2bcf3a5e12d269d8ad7c1a4a8875p-88L +}; + +/* + * Kernel for expl(x). x must be finite and not tiny or huge. + * "tiny" is anything that would make us underflow (|A6*x^6| < ~LDBL_MIN). + * "huge" is anything that would make fn*L1 inexact (|x| > ~2**17*ln2). + */ +static inline void +__k_expl(long double x, long double *hip, long double *lop, int *kp) +{ + long double q, r, r1, t; + double dr, fn, r2; + int n, n2; + + /* Reduce x to (k*ln2 + endpoint[n2] + r1 + r2). */ + /* Use a specialized rint() to get fn. Assume round-to-nearest. */ + /* XXX assume no extra precision for the additions, as for trig fns. */ + /* XXX this set of comments is now quadruplicated. */ + /* XXX but see ../src/e_exp.c for a fix using double_t. */ + fn = (double)x * INV_L + 0x1.8p52 - 0x1.8p52; +#if defined(HAVE_EFFICIENT_IRINT) + n = irint(fn); +#else + n = (int)fn; +#endif + n2 = (unsigned)n % INTERVALS; + /* Depend on the sign bit being propagated: */ + *kp = n >> LOG2_INTERVALS; + r1 = x - fn * L1; + r2 = fn * -L2; + r = r1 + r2; + + /* Evaluate expl(endpoint[n2] + r1 + r2) = tbl[n2] * expl(r1 + r2). */ + dr = r; + q = r2 + r * r * (A2 + r * (A3 + r * (A4 + r * (A5 + r * (A6 + + dr * (A7 + dr * (A8 + dr * (A9 + dr * A10)))))))); + t = tbl[n2].lo + tbl[n2].hi; + *hip = tbl[n2].hi; + *lop = tbl[n2].lo + t * (q + r1); +} + +/* + * XXX: the rest of the functions are identical for ld80 and ld128. + * However, we should use scalbnl() for ld128, since long double + * multiplication is very slow on the only supported ld128 arch (sparc64). + */ + +static inline void +k_hexpl(long double x, long double *hip, long double *lop) +{ + float twopkm1; + int k; + + __k_expl(x, hip, lop, &k); + SET_FLOAT_WORD(twopkm1, 0x3f800000 + ((k - 1) << 23)); + *hip *= twopkm1; + *lop *= twopkm1; +} + +static inline long double +hexpl(long double x) +{ + long double hi, lo, twopkm2; + int k; + + twopkm2 = 1; + __k_expl(x, &hi, &lo, &k); + SET_LDBL_EXPSIGN(twopkm2, BIAS + k - 2); + return (lo + hi) * 2 * twopkm2; +} + +#ifdef _COMPLEX_H +/* + * See ../src/k_exp.c for details. + */ +static inline long double complex +__ldexp_cexpl(long double complex z, int expt) +{ + long double exp_x, hi, lo; + long double x, y, scale1, scale2; + int half_expt, k; + + x = creall(z); + y = cimagl(z); + __k_expl(x, &hi, &lo, &k); + + exp_x = (lo + hi) * 0x1p16382; + expt += k - 16382; + + scale1 = 1; + half_expt = expt / 2; + SET_LDBL_EXPSIGN(scale1, BIAS + half_expt); + scale2 = 1; + SET_LDBL_EXPSIGN(scale1, BIAS + expt - half_expt); + + return (cpackl(cos(y) * exp_x * scale1 * scale2, + sinl(y) * exp_x * scale1 * scale2)); +} +#endif /* _COMPLEX_H */ diff --git a/libm/upstream-freebsd/lib/msun/ld128/k_sinl.c b/libm/upstream-freebsd/lib/msun/ld128/k_sinl.c index bd415c053..09472d65f 100644 --- a/libm/upstream-freebsd/lib/msun/ld128/k_sinl.c +++ b/libm/upstream-freebsd/lib/msun/ld128/k_sinl.c @@ -6,7 +6,7 @@ * * Developed at SunSoft, a Sun Microsystems, Inc. business. * Permission to use, copy, modify, and distribute this - * software is freely granted, provided that this notice + * software is freely granted, provided that this notice * is preserved. * ==================================================== */ diff --git a/libm/upstream-freebsd/lib/msun/ld128/s_erfl.c b/libm/upstream-freebsd/lib/msun/ld128/s_erfl.c new file mode 100644 index 000000000..e29c9691f --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/ld128/s_erfl.c @@ -0,0 +1,329 @@ +/* @(#)s_erf.c 5.1 93/09/24 */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +#include +__FBSDID("$FreeBSD$"); + +/* + * See s_erf.c for complete comments. + * + * Converted to long double by Steven G. Kargl. + */ +#include + +#include "fpmath.h" +#include "math.h" +#include "math_private.h" + +/* XXX Prevent compilers from erroneously constant folding these: */ +static const volatile long double tiny = 0x1p-10000L; + +static const double +half= 0.5, +one = 1, +two = 2; +/* + * In the domain [0, 2**-40], only the first term in the power series + * expansion of erf(x) is used. The magnitude of the first neglected + * terms is less than 2**-120. + */ +static const long double +efx = 1.28379167095512573896158903121545167e-01L, /* 0xecbff6a7, 0x481dd788, 0xb64d21a8, 0xeb06fc3f */ +efx8 = 1.02703333676410059116927122497236133e+00L, /* 0xecbff6a7, 0x481dd788, 0xb64d21a8, 0xeb06ff3f */ +/* + * Domain [0, 0.84375], range ~[-1.919e-38, 1.919e-38]: + * |(erf(x) - x)/x - pp(x)/qq(x)| < 2**-125.29 + */ +pp0 = 1.28379167095512573896158903121545167e-01L, /* 0x3ffc06eb, 0xa8214db6, 0x88d71d48, 0xa7f6bfec */ +pp1 = -3.14931554396568573802046931159683404e-01L, /* 0xbffd427d, 0x6ada7263, 0x547eb096, 0x95f37463 */ +pp2 = -5.27514920282183487103576956956725309e-02L, /* 0xbffab023, 0xe5a271e3, 0xb0e79b01, 0x2f7ac962 */ +pp3 = -1.13202828509005281355609495523452713e-02L, /* 0xbff872f1, 0x6a5023a1, 0xe08b3884, 0x326af20f */ +pp4 = -9.18626155872522453865998391206048506e-04L, /* 0xbff4e19f, 0xea5fb024, 0x43247a37, 0xe430b06c */ +pp5 = -7.87518862406176274922506447157284230e-05L, /* 0xbff14a4f, 0x31a85fe0, 0x7fff2204, 0x09c49b37 */ +pp6 = -3.42357944472240436548115331090560881e-06L, /* 0xbfeccb81, 0x4b43c336, 0xcd2eb6c2, 0x903f2d87 */ +pp7 = -1.37317432573890412634717890726745428e-07L, /* 0xbfe826e3, 0x0e915eb6, 0x42aee414, 0xf7e36805 */ +pp8 = -2.71115170113861755855049008732113726e-09L, /* 0xbfe2749e, 0x2b94fd00, 0xecb4d166, 0x0efb91f8 */ +pp9 = -3.37925756196555959454018189718117864e-11L, /* 0xbfdc293e, 0x1d9060cb, 0xd043204a, 0x314cd7f0 */ +qq1 = 4.76672625471551170489978555182449450e-01L, /* 0x3ffde81c, 0xde6531f0, 0x76803bee, 0x526e29e9 */ +qq2 = 1.06713144672281502058807525850732240e-01L, /* 0x3ffbb518, 0xd7a6bb74, 0xcd9bdd33, 0x7601eee5 */ +qq3 = 1.47747613127513761102189201923147490e-02L, /* 0x3ff8e423, 0xae527e18, 0xf12cb447, 0x723b4749 */ +qq4 = 1.39939377672028671891148770908874816e-03L, /* 0x3ff56ed7, 0xba055d84, 0xc21b45c4, 0x388d1812 */ +qq5 = 9.44302939359455241271983309378738276e-05L, /* 0x3ff18c11, 0xc18c99a4, 0x86d0fe09, 0x46387b4c */ +qq6 = 4.56199342312522842161301671745365650e-06L, /* 0x3fed3226, 0x73421d05, 0x08875300, 0x32fa1432 */ +qq7 = 1.53019260483764773845294600092361197e-07L, /* 0x3fe8489b, 0x3a63f627, 0x2b9ad2ce, 0x26516e57 */ +qq8 = 3.25542691121324805094777901250005508e-09L, /* 0x3fe2bf6c, 0x26d93a29, 0x9142be7c, 0x9f1dd043 */ +qq9 = 3.37405581964478060434410167262684979e-11L; /* 0x3fdc28c8, 0xfb8fa1be, 0x10e57eec, 0xaa19e49f */ + +static const long double +erx = 8.42700792949714894142232424201210961e-01L, /* 0x3ffeaf76, 0x7a741088, 0xb0000000, 0x00000000 */ +/* + * Domain [0.84375, 1.25], range ~[-2.521e-36, 2.523e-36]: + * |(erf(x) - erx) - pa(x)/qa(x)| < 2**-120.15 + */ +pa0 = -2.48010117891186017024438233323795897e-17L, /* 0xbfc7c97f, 0x77812279, 0x6c877f22, 0xef4bfb2e */ +pa1 = 4.15107497420594680894327969504526489e-01L, /* 0x3ffda911, 0xf096fbc2, 0x55662005, 0x2337fa64 */ +pa2 = -3.94180628087084846724448515851892609e-02L, /* 0xbffa42e9, 0xab54528c, 0xad529da1, 0x6efc2af3 */ +pa3 = 4.48897599625192107295954790681677462e-02L, /* 0x3ffa6fbc, 0xa65edba1, 0x0e4cbcea, 0x73ef9a31 */ +pa4 = 8.02069252143016600110972019232995528e-02L, /* 0x3ffb4887, 0x0e8b548e, 0x3230b417, 0x11b553b3 */ +pa5 = -1.02729816533435279443621120242391295e-02L, /* 0xbff850a0, 0x041de3ee, 0xd5bca6c9, 0x4ef5f9f2 */ +pa6 = 5.70777694530755634864821094419982095e-03L, /* 0x3ff77610, 0x9b501e10, 0x4c978382, 0x742df68f */ +pa7 = 1.22635150233075521018231779267077071e-03L, /* 0x3ff5417b, 0x0e623682, 0x60327da0, 0x96b9219e */ +pa8 = 5.36100234820204569428412542856666503e-04L, /* 0x3ff41912, 0x27ceb4c1, 0x1d3298ec, 0x84ced627 */ +pa9 = -1.97753571846365167177187858667583165e-04L, /* 0xbff29eb8, 0x23f5bcf3, 0x15c83c46, 0xe4fda98b */ +pa10 = 6.19333039900846970674794789568415105e-05L, /* 0x3ff103c4, 0x60f88e46, 0xc0c9fb02, 0x13cc7fc1 */ +pa11 = -5.40531400436645861492290270311751349e-06L, /* 0xbfed6abe, 0x9665f8a8, 0xdd0ad3ba, 0xe5dc0ee3 */ +qa1 = 9.05041313265490487793231810291907851e-01L, /* 0x3ffecf61, 0x93340222, 0xe9930620, 0xc4e61168 */ +qa2 = 6.79848064708886864767240880834868092e-01L, /* 0x3ffe5c15, 0x0ba858dc, 0xf7900ae9, 0xfea1e09a */ +qa3 = 4.04720609926471677581066689316516445e-01L, /* 0x3ffd9e6f, 0x145e9b00, 0x6d8c1749, 0xd2928623 */ +qa4 = 1.69183273898369996364661075664302225e-01L, /* 0x3ffc5a7c, 0xc2a363c1, 0xd6c19097, 0xef9b4063 */ +qa5 = 7.44476185988067992342479750486764248e-02L, /* 0x3ffb30ef, 0xfc7259ef, 0x1bcbb089, 0x686dd62d */ +qa6 = 2.02981172725892407200420389604788573e-02L, /* 0x3ff94c90, 0x7976cb0e, 0x21e1d36b, 0x0f09ca2b */ +qa7 = 6.94281866271607668268269403102277234e-03L, /* 0x3ff7c701, 0x2b193250, 0xc5d46ecc, 0x374843d8 */ +qa8 = 1.12952275469171559611651594706820034e-03L, /* 0x3ff52818, 0xfd2a7c06, 0xd13e38fd, 0xda4b34f5 */ +qa9 = 3.13736683241992737197226578597710179e-04L, /* 0x3ff348fa, 0x0cb48d18, 0x051f849b, 0x135ccf74 */ +qa10 = 1.17037675204033225470121134087771410e-05L, /* 0x3fee88b6, 0x98f47704, 0xa5d8f8f2, 0xc6422e11 */ +qa11 = 4.61312518293853991439362806880973592e-06L, /* 0x3fed3594, 0xe31db94f, 0x3592b693, 0xed4386b4 */ +qa12 = -1.02158572037456893687737553657431771e-06L; /* 0xbfeb123a, 0xd60d9b1e, 0x1f6fdeb9, 0x7dc8410a */ +/* + * Domain [1.25,2.85715], range ~[-2.922e-37,2.922e-37]: + * |log(x*erfc(x)) + x**2 + 0.5625 - ra(x)/sa(x)| < 2**-121.36 + */ +static const long double +ra0 = -9.86494292470069009555706994426014461e-03L, /* 0xbff84341, 0x239e8709, 0xe941b06a, 0xcb4b6ec5 */ +ra1 = -1.13580436992565640457579040117568870e+00L, /* 0xbfff22c4, 0x133f7c0d, 0x72d5e231, 0x2eb1ee3f */ +ra2 = -4.89744330295291950661185707066921755e+01L, /* 0xc00487cb, 0xa38b4fc2, 0xc136695b, 0xc1df8047 */ +ra3 = -1.10766149300215937173768072715352140e+03L, /* 0xc00914ea, 0x55e6beb3, 0xabc50e07, 0xb6e5664d */ +ra4 = -1.49991031232170934967642795601952100e+04L, /* 0xc00cd4b8, 0xd33243e6, 0xffbf6545, 0x3c57ef6e */ +ra5 = -1.29805749738318462882524181556996692e+05L, /* 0xc00ffb0d, 0xbfeed9b6, 0x5b2a3ff4, 0xe245bd3c */ +ra6 = -7.42828497044940065828871976644647850e+05L, /* 0xc0126ab5, 0x8fe7caca, 0x473352d9, 0xcd4e0c90 */ +ra7 = -2.85637299581890734287995171242421106e+06L, /* 0xc0145cad, 0xa7f76fe7, 0x3e358051, 0x1799f927 */ +ra8 = -7.40674797129824999383748865571026084e+06L, /* 0xc015c412, 0x6fe29c02, 0x298ad158, 0x7d24e45c */ +ra9 = -1.28653420911930973914078724204151759e+07L, /* 0xc016889e, 0x7c2eb0dc, 0x95d5863b, 0x0aa34dc3 */ +ra10 = -1.47198163599330179552932489109452638e+07L, /* 0xc016c136, 0x90b84923, 0xf9bcb497, 0x19bbd0f5 */ +ra11 = -1.07812992258382800318665248311522624e+07L, /* 0xc0164904, 0xe673a113, 0x35d7f079, 0xe13701f3 */ +ra12 = -4.83545565681708642630419905537756076e+06L, /* 0xc0152721, 0xfea094a8, 0x869eb39d, 0x413d6f13 */ +ra13 = -1.23956521201673964822976917356685286e+06L, /* 0xc0132ea0, 0xd3646baa, 0x2fe62b0d, 0xbae5ce85 */ +ra14 = -1.62289333553652417591275333240371812e+05L, /* 0xc0103cf8, 0xaab1e2d6, 0x4c25e014, 0x248d76ab */ +ra15 = -8.82890392601176969729168894389833110e+03L, /* 0xc00c13e7, 0x3b3d8f94, 0x6fbda6f6, 0xe7049a82 */ +ra16 = -1.22591866337261720023681535568334619e+02L, /* 0xc005ea5e, 0x12358891, 0xcfa712c5, 0x77f050d4 */ +sa1 = 6.44508918884710829371852723353794047e+01L, /* 0x400501cd, 0xb69a6c0f, 0x5716de14, 0x47161af6 */ +sa2 = 1.76118475473171481523704824327358534e+03L, /* 0x4009b84b, 0xd305829f, 0xc4c771b0, 0xbf1f7f9b */ +sa3 = 2.69448346969488374857087646131950188e+04L, /* 0x400da503, 0x56bacc05, 0x4fdba68d, 0x2cca27e6 */ +sa4 = 2.56826633369941456778326497384543763e+05L, /* 0x4010f59d, 0x51124428, 0x69c41de6, 0xbd0d5753 */ +sa5 = 1.60647413092257206847700054645905859e+06L, /* 0x40138834, 0xa2184244, 0x557a1bed, 0x68c9d556 */ +sa6 = 6.76963075165099718574753447122393797e+06L, /* 0x40159d2f, 0x7b01b0cc, 0x8bac9e95, 0x5d35d56e */ +sa7 = 1.94295690905361884290986932493647741e+07L, /* 0x40172878, 0xc1172d61, 0x3068501e, 0x2f3c71da */ +sa8 = 3.79774781017759149060839255547073541e+07L, /* 0x401821be, 0xc30d06fe, 0x410563d7, 0x032111fd */ +sa9 = 5.00659831846029484248302236457727397e+07L, /* 0x40187df9, 0x1f97a111, 0xc51d6ac2, 0x4b389793 */ +sa10 = 4.36486287620506484276130525941972541e+07L, /* 0x40184d03, 0x3a618ae0, 0x2a723357, 0xfa45c60a */ +sa11 = 2.43779678791333894255510508253951934e+07L, /* 0x401773fa, 0x6fe10ee2, 0xc467850d, 0xc6b7ff30 */ +sa12 = 8.30732360384443202039372372212966542e+06L, /* 0x4015fb09, 0xee6a5631, 0xdd98de7e, 0x8b00461a */ +sa13 = 1.60160846942050515734192397495105693e+06L, /* 0x40138704, 0x8782bf13, 0x5b8fb315, 0xa898abe5 */ +sa14 = 1.54255505242533291014555153757001825e+05L, /* 0x40102d47, 0xc0abc98e, 0x843c9490, 0xb4352440 */ +sa15 = 5.87949220002375547561467275493888824e+03L, /* 0x400b6f77, 0xe00d21d1, 0xec4d41e8, 0x2f8e1673 */ +sa16 = 4.97272976346793193860385983372237710e+01L; /* 0x40048dd1, 0x816c1b3f, 0x24f540a6, 0x4cfe03cc */ +/* + * Domain [2.85715,9], range ~[-7.886e-37,7.918e-37]: + * |log(x*erfc(x)) + x**2 + 0.5625 - rb(x)/sb(x)| < 2**-120 + */ +static const long double +rb0 = -9.86494292470008707171371994479162369e-3L, /* 0xbff84341, 0x239e86f4, 0x2f57e561, 0xf4469360 */ +rb1 = -1.57047326624110727986326503729442830L, /* 0xbfff920a, 0x8935bf73, 0x8803b894, 0x4656482d */ +rb2 = -1.03228196364885474342132255440317065e2L, /* 0xc0059ce9, 0xac4ed0ff, 0x2cff0ff7, 0x5e70d1ab */ +rb3 = -3.74000570653418227179358710865224376e3L, /* 0xc00ad380, 0x2ebf7835, 0xf6b07ed2, 0x861242f7 */ +rb4 = -8.35435477739098044190860390632813956e4L, /* 0xc00f4657, 0x8c3ae934, 0x3647d7b3, 0x80e76fb7 */ +rb5 = -1.21398672055223642118716640216747152e6L, /* 0xc0132862, 0x2b8761c8, 0x27d18c0f, 0x137c9463 */ +rb6 = -1.17669175877248796101665344873273970e7L, /* 0xc0166719, 0x0b2cea46, 0x81f14174, 0x11602ea5 */ +rb7 = -7.66108006086998253606773064264599615e7L, /* 0xc019243f, 0x3c26f4f0, 0x1cc05241, 0x3b953728 */ +rb8 = -3.32547117558141845968704725353130804e8L, /* 0xc01b3d24, 0x42d8ee26, 0x24ef6f3b, 0x604a8c65 */ +rb9 = -9.41561252426350696802167711221739746e8L, /* 0xc01cc0f8, 0xad23692a, 0x8ddb2310, 0xe9937145 */ +rb10 = -1.67157110805390944549427329626281063e9L, /* 0xc01d8e88, 0x9a903734, 0x09a55fa3, 0xd205c903 */ +rb11 = -1.74339631004410841337645931421427373e9L, /* 0xc01d9fa8, 0x77582d2a, 0xc183b8ab, 0x7e00cb05 */ +rb12 = -9.57655233596934915727573141357471703e8L, /* 0xc01cc8a5, 0x460cc685, 0xd0271fa0, 0x6a70e3da */ +rb13 = -2.26320062731339353035254704082495066e8L, /* 0xc01aafab, 0xd7d76721, 0xc9720e11, 0x6a8bd489 */ +rb14 = -1.42777302996263256686002973851837039e7L, /* 0xc016b3b8, 0xc499689f, 0x2b88d965, 0xc32414f9 */ +sb1 = 1.08512869705594540211033733976348506e2L, /* 0x4005b20d, 0x2db7528d, 0x00d20dcb, 0x858f6191 */ +sb2 = 5.02757713761390460534494530537572834e3L, /* 0x400b3a39, 0x3bf4a690, 0x3025d28d, 0xfd40a891 */ +sb3 = 1.31019107205412870059331647078328430e5L, /* 0x400fffcb, 0x1b71d05e, 0x3b28361d, 0x2a3c3690 */ +sb4 = 2.13021555152296846166736757455018030e6L, /* 0x40140409, 0x3c6984df, 0xc4491d7c, 0xb04aa08d */ +sb5 = 2.26649105281820861953868568619768286e7L, /* 0x401759d6, 0xce8736f0, 0xf28ad037, 0x2a901e0c */ +sb6 = 1.61071939490875921812318684143076081e8L, /* 0x401a3338, 0x686fb541, 0x6bd27d06, 0x4f95c9ac */ +sb7 = 7.66895673844301852676056750497991966e8L, /* 0x401c6daf, 0x31cec121, 0x54699126, 0x4bd9bf9e */ +sb8 = 2.41884450436101936436023058196042526e9L, /* 0x401e2059, 0x46b0b8d7, 0x87b64cbf, 0x78bc296d */ +sb9 = 4.92403055884071695093305291535107666e9L, /* 0x401f257e, 0xbe5ed739, 0x39e17346, 0xcadd2e55 */ +sb10 = 6.18627786365587486459633615573786416e9L, /* 0x401f70bb, 0x1be7a7e7, 0x6a45b5ae, 0x607c70f0 */ +sb11 = 4.45898013426501378097430226324743199e9L, /* 0x401f09c6, 0xa32643d7, 0xf1724620, 0x9ea46c32 */ +sb12 = 1.63006115763329848117160344854224975e9L, /* 0x401d84a3, 0x0996887f, 0x65a4f43b, 0x978c1d74 */ +sb13 = 2.39216717012421697446304015847567721e8L, /* 0x401ac845, 0x09a065c2, 0x30095da7, 0x9d72d6ae */ +sb14 = 7.84837329009278694937250358810225609e6L; /* 0x4015df06, 0xd5290e15, 0x63031fac, 0x4d9c894c */ +/* + * Domain [9,108], range ~[-5.324e-38,5.340e-38]: + * |log(x*erfc(x)) + x**2 + 0.5625 - r(x)/s(x)| < 2**-124 + */ +static const long double +rc0 = -9.86494292470008707171367567652935673e-3L, /* 0xbff84341, 0x239e86f4, 0x2f57e55b, 0x1aa10fd3 */ +rc1 = -1.26229447747315096406518846411562266L, /* 0xbfff4325, 0xbb1aab28, 0xda395cd9, 0xfb861c15 */ +rc2 = -6.13742634438922591780742637728666162e1L, /* 0xc004eafe, 0x7dd51cd8, 0x3c7c5928, 0x751e50cf */ +rc3 = -1.50455835478908280402912854338421517e3L, /* 0xc0097823, 0xbc15b9ab, 0x3d60745c, 0x523e80a5 */ +rc4 = -2.04415631865861549920184039902945685e4L, /* 0xc00d3f66, 0x40b3fc04, 0x5388f2ec, 0xb009e1f0 */ +rc5 = -1.57625662981714582753490610560037638e5L, /* 0xc01033dc, 0xd4dc95b6, 0xfd4da93b, 0xf355b4a9 */ +rc6 = -6.73473451616752528402917538033283794e5L, /* 0xc01248d8, 0x2e73a4f9, 0xcded49c5, 0xfa3bfeb7 */ +rc7 = -1.47433165421387483167186683764364857e6L, /* 0xc01367f1, 0xba77a8f7, 0xcfdd0dbb, 0x25d554b3 */ +rc8 = -1.38811981807868828563794929997744139e6L, /* 0xc01352e5, 0x7d16d9ad, 0xbbdcbf38, 0x38fbc5ea */ +rc9 = -3.59659700530831825640766479698155060e5L, /* 0xc0115f3a, 0xecd57f45, 0x21f8ad6c, 0x910a5958 */ +sc1 = 7.72730753022908298637508998072635696e1L, /* 0x40053517, 0xa10d52bc, 0xdabb55b6, 0xbd0328cd */ +sc2 = 2.36825757341694050500333261769082182e3L, /* 0x400a2808, 0x3e0a9b42, 0x82977842, 0x9c5de29e */ +sc3 = 3.72210540173034735352888847134073099e4L, /* 0x400e22ca, 0x1ba827ef, 0xac8390d7, 0x1fc39a41 */ +sc4 = 3.24136032646418336712461033591393412e5L, /* 0x40113c8a, 0x0216e100, 0xc59d1e44, 0xf0e68d9d */ +sc5 = 1.57836135851134393802505823370009175e6L, /* 0x40138157, 0x95bc7664, 0x17575961, 0xdbe58eeb */ +sc6 = 4.12881981392063738026679089714182355e6L, /* 0x4014f801, 0x9e82e8d2, 0xb8b3a70e, 0xfd84185d */ +sc7 = 5.24438427289213488410596395361544142e6L, /* 0x40154017, 0x81177109, 0x2aa6c3b0, 0x1f106625 */ +sc8 = 2.59909544563616121735963429710382149e6L, /* 0x40143d45, 0xbb90a9b1, 0x12bf9390, 0xa827a700 */ +sc9 = 2.80930665169282501639651995082335693e5L; /* 0x40111258, 0xaa92222e, 0xa97e3216, 0xa237fa6c */ + +long double +erfl(long double x) +{ + long double ax,R,S,P,Q,s,y,z,r; + uint64_t lx, llx; + int32_t i; + uint16_t hx; + + EXTRACT_LDBL128_WORDS(hx, lx, llx, x); + + if((hx & 0x7fff) == 0x7fff) { /* erfl(nan)=nan */ + i = (hx>>15)<<1; + return (1-i)+one/x; /* erfl(+-inf)=+-1 */ + } + + ax = fabsl(x); + if(ax < 0.84375) { + if(ax < 0x1p-40L) { + if(ax < 0x1p-16373L) + return (8*x+efx8*x)/8; /* avoid spurious underflow */ + return x + efx*x; + } + z = x*x; + r = pp0+z*(pp1+z*(pp2+z*(pp3+z*(pp4+z*(pp5+z*(pp6+z*(pp7+ + z*(pp8+z*pp9)))))))); + s = one+z*(qq1+z*(qq2+z*(qq3+z*(qq4+z*(qq5+z*(qq6+z*(qq7+ + z*(qq8+z*qq9)))))))); + y = r/s; + return x + x*y; + } + if(ax < 1.25) { + s = ax-one; + P = pa0+s*(pa1+s*(pa2+s*(pa3+s*(pa4+s*(pa5+s*(pa6+s*(pa7+ + s*(pa8+s*(pa9+s*(pa10+s*pa11)))))))))); + Q = one+s*(qa1+s*(qa2+s*(qa3+s*(qa4+s*(qa5+s*(qa6+s*(qa7+ + s*(qa8+s*(qa9+s*(qa10+s*(qa11+s*qa12))))))))))); + if(x>=0) return (erx + P/Q); else return (-erx - P/Q); + } + if (ax >= 9) { /* inf>|x|>= 9 */ + if(x>=0) return (one-tiny); else return (tiny-one); + } + s = one/(ax*ax); + if(ax < 2.85715) { /* |x| < 2.85715 */ + R=ra0+s*(ra1+s*(ra2+s*(ra3+s*(ra4+s*(ra5+s*(ra6+s*(ra7+ + s*(ra8+s*(ra9+s*(ra10+s*(ra11+s*(ra12+s*(ra13+s*(ra14+ + s*(ra15+s*ra16))))))))))))))); + S=one+s*(sa1+s*(sa2+s*(sa3+s*(sa4+s*(sa5+s*(sa6+s*(sa7+ + s*(sa8+s*(sa9+s*(sa10+s*(sa11+s*(sa12+s*(sa13+s*(sa14+ + s*(sa15+s*sa16))))))))))))))); + } else { /* |x| >= 2.85715 */ + R=rb0+s*(rb1+s*(rb2+s*(rb3+s*(rb4+s*(rb5+s*(rb6+s*(rb7+ + s*(rb8+s*(rb9+s*(rb10+s*(rb11+s*(rb12+s*(rb13+ + s*rb14))))))))))))); + S=one+s*(sb1+s*(sb2+s*(sb3+s*(sb4+s*(sb5+s*(sb6+s*(sb7+ + s*(sb8+s*(sb9+s*(sb10+s*(sb11+s*(sb12+s*(sb13+ + s*sb14))))))))))))); + } + z = (float)ax; + r = expl(-z*z-0.5625)*expl((z-ax)*(z+ax)+R/S); + if(x>=0) return (one-r/ax); else return (r/ax-one); +} + +long double +erfcl(long double x) +{ + long double ax,R,S,P,Q,s,y,z,r; + uint64_t lx, llx; + uint16_t hx; + + EXTRACT_LDBL128_WORDS(hx, lx, llx, x); + + if((hx & 0x7fff) == 0x7fff) { /* erfcl(nan)=nan */ + /* erfcl(+-inf)=0,2 */ + return ((hx>>15)<<1)+one/x; + } + + ax = fabsl(x); + if(ax < 0.84375L) { + if(ax < 0x1p-34L) + return one-x; + z = x*x; + r = pp0+z*(pp1+z*(pp2+z*(pp3+z*(pp4+z*(pp5+z*(pp6+z*(pp7+ + z*(pp8+z*pp9)))))))); + s = one+z*(qq1+z*(qq2+z*(qq3+z*(qq4+z*(qq5+z*(qq6+z*(qq7+ + z*(qq8+z*qq9)))))))); + y = r/s; + if(ax < 0.25L) { /* x<1/4 */ + return one-(x+x*y); + } else { + r = x*y; + r += (x-half); + return half - r; + } + } + if(ax < 1.25L) { + s = ax-one; + P = pa0+s*(pa1+s*(pa2+s*(pa3+s*(pa4+s*(pa5+s*(pa6+s*(pa7+ + s*(pa8+s*(pa9+s*(pa10+s*pa11)))))))))); + Q = one+s*(qa1+s*(qa2+s*(qa3+s*(qa4+s*(qa5+s*(qa6+s*(qa7+ + s*(qa8+s*(qa9+s*(qa10+s*(qa11+s*qa12))))))))))); + if(x>=0) { + z = one-erx; return z - P/Q; + } else { + z = erx+P/Q; return one+z; + } + } + + if(ax < 108) { /* |x| < 108 */ + s = one/(ax*ax); + if(ax < 2.85715) { /* |x| < 2.85715 */ + R=ra0+s*(ra1+s*(ra2+s*(ra3+s*(ra4+s*(ra5+s*(ra6+s*(ra7+ + s*(ra8+s*(ra9+s*(ra10+s*(ra11+s*(ra12+s*(ra13+s*(ra14+ + s*(ra15+s*ra16))))))))))))))); + S=one+s*(sa1+s*(sa2+s*(sa3+s*(sa4+s*(sa5+s*(sa6+s*(sa7+ + s*(sa8+s*(sa9+s*(sa10+s*(sa11+s*(sa12+s*(sa13+s*(sa14+ + s*(sa15+s*sa16))))))))))))))); + } else if(ax < 9) { + R=rb0+s*(rb1+s*(rb2+s*(rb3+s*(rb4+s*(rb5+s*(rb6+s*(rb7+ + s*(rb8+s*(rb9+s*(rb10+s*(rb11+s*(rb12+s*(rb13+ + s*rb14))))))))))))); + S=one+s*(sb1+s*(sb2+s*(sb3+s*(sb4+s*(sb5+s*(sb6+s*(sb7+ + s*(sb8+s*(sb9+s*(sb10+s*(sb11+s*(sb12+s*(sb13+ + s*sb14))))))))))))); + } else { + if(x < -9) return two-tiny; /* x < -9 */ + R=rc0+s*(rc1+s*(rc2+s*(rc3+s*(rc4+s*(rc5+s*(rc6+s*(rc7+ + s*(rc8+s*rc9)))))))); + S=one+s*(sc1+s*(sc2+s*(sc3+s*(sc4+s*(sc5+s*(sc6+s*(sc7+ + s*(sc8+s*sc9)))))))); + } + z = (float)ax; + r = expl(-z*z-0.5625)*expl((z-ax)*(z+ax)+R/S); + if(x>0) return r/ax; else return two-r/ax; + } else { + if(x>0) return tiny*tiny; else return two-tiny; + } +} diff --git a/libm/upstream-freebsd/lib/msun/ld128/s_exp2l.c b/libm/upstream-freebsd/lib/msun/ld128/s_exp2l.c index 5ed514c97..5afa37e58 100644 --- a/libm/upstream-freebsd/lib/msun/ld128/s_exp2l.c +++ b/libm/upstream-freebsd/lib/msun/ld128/s_exp2l.c @@ -369,7 +369,7 @@ exp2l(long double x) || u.xbits.manl != 0 || (hx & 0x8000) == 0) return (x + x); /* x is NaN or +Inf */ - else + else return (0.0); /* x is -Inf */ } if (x >= 16384) diff --git a/libm/upstream-freebsd/lib/msun/ld128/s_expl.c b/libm/upstream-freebsd/lib/msun/ld128/s_expl.c index 176c93255..a6a967675 100644 --- a/libm/upstream-freebsd/lib/msun/ld128/s_expl.c +++ b/libm/upstream-freebsd/lib/msun/ld128/s_expl.c @@ -38,16 +38,15 @@ __FBSDID("$FreeBSD$"); #include "fpmath.h" #include "math.h" #include "math_private.h" +#include "k_expl.h" -#define INTERVALS 128 -#define LOG2_INTERVALS 7 -#define BIAS (LDBL_MAX_EXP - 1) +/* XXX Prevent compilers from erroneously constant folding these: */ +static const volatile long double +huge = 0x1p10000L, +tiny = 0x1p-10000L; static const long double -huge = 0x1p10000L, twom10000 = 0x1p-10000L; -/* XXX Prevent gcc from erroneously constant folding this: */ -static volatile const long double tiny = 0x1p-10000L; static const long double /* log(2**16384 - 0.5) rounded towards zero: */ @@ -56,184 +55,16 @@ o_threshold = 11356.523406294143949491931077970763428L, /* log(2**(-16381-64-1)) rounded towards zero: */ u_threshold = -11433.462743336297878837243843452621503L; -static const double -/* - * ln2/INTERVALS = L1+L2 (hi+lo decomposition for multiplication). L1 must - * have at least 22 (= log2(|LDBL_MIN_EXP-extras|) + log2(INTERVALS)) lowest - * bits zero so that multiplication of it by n is exact. - */ -INV_L = 1.8466496523378731e+2, /* 0x171547652b82fe.0p-45 */ -L2 = -1.0253670638894731e-29; /* -0x1.9ff0342542fc3p-97 */ -static const long double -/* 0x1.62e42fefa39ef35793c768000000p-8 */ -L1 = 5.41521234812457272982212595914567508e-3L; - -static const long double -/* - * Domain [-0.002708, 0.002708], range ~[-2.4021e-38, 2.4234e-38]: - * |exp(x) - p(x)| < 2**-124.9 - * (0.002708 is ln2/(2*INTERVALS) rounded up a little). - */ -A2 = 0.5, -A3 = 1.66666666666666666666666666651085500e-1L, -A4 = 4.16666666666666666666666666425885320e-2L, -A5 = 8.33333333333333333334522877160175842e-3L, -A6 = 1.38888888888888888889971139751596836e-3L; - -static const double -A7 = 1.9841269841269471e-4, -A8 = 2.4801587301585284e-5, -A9 = 2.7557324277411234e-6, -A10 = 2.7557333722375072e-7; - -static const struct { - /* - * hi must be rounded to at most 106 bits so that multiplication - * by r1 in expm1l() is exact, but it is rounded to 88 bits due to - * historical accidents. - */ - long double hi; - long double lo; -} tbl[INTERVALS] = { - 0x1p0L, 0x0p0L, - 0x1.0163da9fb33356d84a66aep0L, 0x3.36dcdfa4003ec04c360be2404078p-92L, - 0x1.02c9a3e778060ee6f7cacap0L, 0x4.f7a29bde93d70a2cabc5cb89ba10p-92L, - 0x1.04315e86e7f84bd738f9a2p0L, 0xd.a47e6ed040bb4bfc05af6455e9b8p-96L, - 0x1.059b0d31585743ae7c548ep0L, 0xb.68ca417fe53e3495f7df4baf84a0p-92L, - 0x1.0706b29ddf6ddc6dc403a8p0L, 0x1.d87b27ed07cb8b092ac75e311753p-88L, - 0x1.0874518759bc808c35f25cp0L, 0x1.9427fa2b041b2d6829d8993a0d01p-88L, - 0x1.09e3ecac6f3834521e060cp0L, 0x5.84d6b74ba2e023da730e7fccb758p-92L, - 0x1.0b5586cf9890f6298b92b6p0L, 0x1.1842a98364291408b3ceb0a2a2bbp-88L, - 0x1.0cc922b7247f7407b705b8p0L, 0x9.3dc5e8aac564e6fe2ef1d431fd98p-92L, - 0x1.0e3ec32d3d1a2020742e4ep0L, 0x1.8af6a552ac4b358b1129e9f966a4p-88L, - 0x1.0fb66affed31af232091dcp0L, 0x1.8a1426514e0b627bda694a400a27p-88L, - 0x1.11301d0125b50a4ebbf1aep0L, 0xd.9318ceac5cc47ab166ee57427178p-92L, - 0x1.12abdc06c31cbfb92bad32p0L, 0x4.d68e2f7270bdf7cedf94eb1cb818p-92L, - 0x1.1429aaea92ddfb34101942p0L, 0x1.b2586d01844b389bea7aedd221d4p-88L, - 0x1.15a98c8a58e512480d573cp0L, 0x1.d5613bf92a2b618ee31b376c2689p-88L, - 0x1.172b83c7d517adcdf7c8c4p0L, 0x1.0eb14a792035509ff7d758693f24p-88L, - 0x1.18af9388c8de9bbbf70b9ap0L, 0x3.c2505c97c0102e5f1211941d2840p-92L, - 0x1.1a35beb6fcb753cb698f68p0L, 0x1.2d1c835a6c30724d5cfae31b84e5p-88L, - 0x1.1bbe084045cd39ab1e72b4p0L, 0x4.27e35f9acb57e473915519a1b448p-92L, - 0x1.1d4873168b9aa7805b8028p0L, 0x9.90f07a98b42206e46166cf051d70p-92L, - 0x1.1ed5022fcd91cb8819ff60p0L, 0x1.121d1e504d36c47474c9b7de6067p-88L, - 0x1.2063b88628cd63b8eeb028p0L, 0x1.50929d0fc487d21c2b84004264dep-88L, - 0x1.21f49917ddc962552fd292p0L, 0x9.4bdb4b61ea62477caa1dce823ba0p-92L, - 0x1.2387a6e75623866c1fadb0p0L, 0x1.c15cb593b0328566902df69e4de2p-88L, - 0x1.251ce4fb2a63f3582ab7dep0L, 0x9.e94811a9c8afdcf796934bc652d0p-92L, - 0x1.26b4565e27cdd257a67328p0L, 0x1.d3b249dce4e9186ddd5ff44e6b08p-92L, - 0x1.284dfe1f5638096cf15cf0p0L, 0x3.ca0967fdaa2e52d7c8106f2e262cp-92L, - 0x1.29e9df51fdee12c25d15f4p0L, 0x1.a24aa3bca890ac08d203fed80a07p-88L, - 0x1.2b87fd0dad98ffddea4652p0L, 0x1.8fcab88442fdc3cb6de4519165edp-88L, - 0x1.2d285a6e4030b40091d536p0L, 0xd.075384589c1cd1b3e4018a6b1348p-92L, - 0x1.2ecafa93e2f5611ca0f45cp0L, 0x1.523833af611bdcda253c554cf278p-88L, - 0x1.306fe0a31b7152de8d5a46p0L, 0x3.05c85edecbc27343629f502f1af2p-92L, - 0x1.32170fc4cd8313539cf1c2p0L, 0x1.008f86dde3220ae17a005b6412bep-88L, - 0x1.33c08b26416ff4c9c8610cp0L, 0x1.96696bf95d1593039539d94d662bp-88L, - 0x1.356c55f929ff0c94623476p0L, 0x3.73af38d6d8d6f9506c9bbc93cbc0p-92L, - 0x1.371a7373aa9caa7145502ep0L, 0x1.4547987e3e12516bf9c699be432fp-88L, - 0x1.38cae6d05d86585a9cb0d8p0L, 0x1.bed0c853bd30a02790931eb2e8f0p-88L, - 0x1.3a7db34e59ff6ea1bc9298p0L, 0x1.e0a1d336163fe2f852ceeb134067p-88L, - 0x1.3c32dc313a8e484001f228p0L, 0xb.58f3775e06ab66353001fae9fca0p-92L, - 0x1.3dea64c12342235b41223ep0L, 0x1.3d773fba2cb82b8244267c54443fp-92L, - 0x1.3fa4504ac801ba0bf701aap0L, 0x4.1832fb8c1c8dbdff2c49909e6c60p-92L, - 0x1.4160a21f72e29f84325b8ep0L, 0x1.3db61fb352f0540e6ba05634413ep-88L, - 0x1.431f5d950a896dc7044394p0L, 0x1.0ccec81e24b0caff7581ef4127f7p-92L, - 0x1.44e086061892d03136f408p0L, 0x1.df019fbd4f3b48709b78591d5cb5p-88L, - 0x1.46a41ed1d005772512f458p0L, 0x1.229d97df404ff21f39c1b594d3a8p-88L, - 0x1.486a2b5c13cd013c1a3b68p0L, 0x1.062f03c3dd75ce8757f780e6ec99p-88L, - 0x1.4a32af0d7d3de672d8bcf4p0L, 0x6.f9586461db1d878b1d148bd3ccb8p-92L, - 0x1.4bfdad5362a271d4397afep0L, 0xc.42e20e0363ba2e159c579f82e4b0p-92L, - 0x1.4dcb299fddd0d63b36ef1ap0L, 0x9.e0cc484b25a5566d0bd5f58ad238p-92L, - 0x1.4f9b2769d2ca6ad33d8b68p0L, 0x1.aa073ee55e028497a329a7333dbap-88L, - 0x1.516daa2cf6641c112f52c8p0L, 0x4.d822190e718226177d7608d20038p-92L, - 0x1.5342b569d4f81df0a83c48p0L, 0x1.d86a63f4e672a3e429805b049465p-88L, - 0x1.551a4ca5d920ec52ec6202p0L, 0x4.34ca672645dc6c124d6619a87574p-92L, - 0x1.56f4736b527da66ecb0046p0L, 0x1.64eb3c00f2f5ab3d801d7cc7272dp-88L, - 0x1.58d12d497c7fd252bc2b72p0L, 0x1.43bcf2ec936a970d9cc266f0072fp-88L, - 0x1.5ab07dd48542958c930150p0L, 0x1.91eb345d88d7c81280e069fbdb63p-88L, - 0x1.5c9268a5946b701c4b1b80p0L, 0x1.6986a203d84e6a4a92f179e71889p-88L, - 0x1.5e76f15ad21486e9be4c20p0L, 0x3.99766a06548a05829e853bdb2b52p-92L, - 0x1.605e1b976dc08b076f592ap0L, 0x4.86e3b34ead1b4769df867b9c89ccp-92L, - 0x1.6247eb03a5584b1f0fa06ep0L, 0x1.d2da42bb1ceaf9f732275b8aef30p-88L, - 0x1.6434634ccc31fc76f8714cp0L, 0x4.ed9a4e41000307103a18cf7a6e08p-92L, - 0x1.66238825522249127d9e28p0L, 0x1.b8f314a337f4dc0a3adf1787ff74p-88L, - 0x1.68155d44ca973081c57226p0L, 0x1.b9f32706bfe4e627d809a85dcc66p-88L, - 0x1.6a09e667f3bcc908b2fb12p0L, 0x1.66ea957d3e3adec17512775099dap-88L, - 0x1.6c012750bdabeed76a9980p0L, 0xf.4f33fdeb8b0ecd831106f57b3d00p-96L, - 0x1.6dfb23c651a2ef220e2cbep0L, 0x1.bbaa834b3f11577ceefbe6c1c411p-92L, - 0x1.6ff7df9519483cf87e1b4ep0L, 0x1.3e213bff9b702d5aa477c12523cep-88L, - 0x1.71f75e8ec5f73dd2370f2ep0L, 0xf.0acd6cb434b562d9e8a20adda648p-92L, - 0x1.73f9a48a58173bd5c9a4e6p0L, 0x8.ab1182ae217f3a7681759553e840p-92L, - 0x1.75feb564267c8bf6e9aa32p0L, 0x1.a48b27071805e61a17b954a2dad8p-88L, - 0x1.780694fde5d3f619ae0280p0L, 0x8.58b2bb2bdcf86cd08e35fb04c0f0p-92L, - 0x1.7a11473eb0186d7d51023ep0L, 0x1.6cda1f5ef42b66977960531e821bp-88L, - 0x1.7c1ed0130c1327c4933444p0L, 0x1.937562b2dc933d44fc828efd4c9cp-88L, - 0x1.7e2f336cf4e62105d02ba0p0L, 0x1.5797e170a1427f8fcdf5f3906108p-88L, - 0x1.80427543e1a11b60de6764p0L, 0x9.a354ea706b8e4d8b718a672bf7c8p-92L, - 0x1.82589994cce128acf88afap0L, 0xb.34a010f6ad65cbbac0f532d39be0p-92L, - 0x1.8471a4623c7acce52f6b96p0L, 0x1.c64095370f51f48817914dd78665p-88L, - 0x1.868d99b4492ec80e41d90ap0L, 0xc.251707484d73f136fb5779656b70p-92L, - 0x1.88ac7d98a669966530bcdep0L, 0x1.2d4e9d61283ef385de170ab20f96p-88L, - 0x1.8ace5422aa0db5ba7c55a0p0L, 0x1.92c9bb3e6ed61f2733304a346d8fp-88L, - 0x1.8cf3216b5448bef2aa1cd0p0L, 0x1.61c55d84a9848f8c453b3ca8c946p-88L, - 0x1.8f1ae991577362b982745cp0L, 0x7.2ed804efc9b4ae1458ae946099d4p-92L, - 0x1.9145b0b91ffc588a61b468p0L, 0x1.f6b70e01c2a90229a4c4309ea719p-88L, - 0x1.93737b0cdc5e4f4501c3f2p0L, 0x5.40a22d2fc4af581b63e8326efe9cp-92L, - 0x1.95a44cbc8520ee9b483694p0L, 0x1.a0fc6f7c7d61b2b3a22a0eab2cadp-88L, - 0x1.97d829fde4e4f8b9e920f8p0L, 0x1.1e8bd7edb9d7144b6f6818084cc7p-88L, - 0x1.9a0f170ca07b9ba3109b8cp0L, 0x4.6737beb19e1eada6825d3c557428p-92L, - 0x1.9c49182a3f0901c7c46b06p0L, 0x1.1f2be58ddade50c217186c90b457p-88L, - 0x1.9e86319e323231824ca78ep0L, 0x6.4c6e010f92c082bbadfaf605cfd4p-92L, - 0x1.a0c667b5de564b29ada8b8p0L, 0xc.ab349aa0422a8da7d4512edac548p-92L, - 0x1.a309bec4a2d3358c171f76p0L, 0x1.0daad547fa22c26d168ea762d854p-88L, - 0x1.a5503b23e255c8b424491cp0L, 0xa.f87bc8050a405381703ef7caff50p-92L, - 0x1.a799e1330b3586f2dfb2b0p0L, 0x1.58f1a98796ce8908ae852236ca94p-88L, - 0x1.a9e6b5579fdbf43eb243bcp0L, 0x1.ff4c4c58b571cf465caf07b4b9f5p-88L, - 0x1.ac36bbfd3f379c0db966a2p0L, 0x1.1265fc73e480712d20f8597a8e7bp-88L, - 0x1.ae89f995ad3ad5e8734d16p0L, 0x1.73205a7fbc3ae675ea440b162d6cp-88L, - 0x1.b0e07298db66590842acdep0L, 0x1.c6f6ca0e5dcae2aafffa7a0554cbp-88L, - 0x1.b33a2b84f15faf6bfd0e7ap0L, 0x1.d947c2575781dbb49b1237c87b6ep-88L, - 0x1.b59728de559398e3881110p0L, 0x1.64873c7171fefc410416be0a6525p-88L, - 0x1.b7f76f2fb5e46eaa7b081ap0L, 0xb.53c5354c8903c356e4b625aacc28p-92L, - 0x1.ba5b030a10649840cb3c6ap0L, 0xf.5b47f297203757e1cc6eadc8bad0p-92L, - 0x1.bcc1e904bc1d2247ba0f44p0L, 0x1.b3d08cd0b20287092bd59be4ad98p-88L, - 0x1.bf2c25bd71e088408d7024p0L, 0x1.18e3449fa073b356766dfb568ff4p-88L, - 0x1.c199bdd85529c2220cb12ap0L, 0x9.1ba6679444964a36661240043970p-96L, - 0x1.c40ab5fffd07a6d14df820p0L, 0xf.1828a5366fd387a7bdd54cdf7300p-92L, - 0x1.c67f12e57d14b4a2137fd2p0L, 0xf.2b301dd9e6b151a6d1f9d5d5f520p-96L, - 0x1.c8f6d9406e7b511acbc488p0L, 0x5.c442ddb55820171f319d9e5076a8p-96L, - 0x1.cb720dcef90691503cbd1ep0L, 0x9.49db761d9559ac0cb6dd3ed599e0p-92L, - 0x1.cdf0b555dc3f9c44f8958ep0L, 0x1.ac51be515f8c58bdfb6f5740a3a4p-88L, - 0x1.d072d4a07897b8d0f22f20p0L, 0x1.a158e18fbbfc625f09f4cca40874p-88L, - 0x1.d2f87080d89f18ade12398p0L, 0x9.ea2025b4c56553f5cdee4c924728p-92L, - 0x1.d5818dcfba48725da05aeap0L, 0x1.66e0dca9f589f559c0876ff23830p-88L, - 0x1.d80e316c98397bb84f9d04p0L, 0x8.805f84bec614de269900ddf98d28p-92L, - 0x1.da9e603db3285708c01a5ap0L, 0x1.6d4c97f6246f0ec614ec95c99392p-88L, - 0x1.dd321f301b4604b695de3cp0L, 0x6.30a393215299e30d4fb73503c348p-96L, - 0x1.dfc97337b9b5eb968cac38p0L, 0x1.ed291b7225a944efd5bb5524b927p-88L, - 0x1.e264614f5a128a12761fa0p0L, 0x1.7ada6467e77f73bf65e04c95e29dp-88L, - 0x1.e502ee78b3ff6273d13014p0L, 0x1.3991e8f49659e1693be17ae1d2f9p-88L, - 0x1.e7a51fbc74c834b548b282p0L, 0x1.23786758a84f4956354634a416cep-88L, - 0x1.ea4afa2a490d9858f73a18p0L, 0xf.5db301f86dea20610ceee13eb7b8p-92L, - 0x1.ecf482d8e67f08db0312fap0L, 0x1.949cef462010bb4bc4ce72a900dfp-88L, - 0x1.efa1bee615a27771fd21a8p0L, 0x1.2dac1f6dd5d229ff68e46f27e3dfp-88L, - 0x1.f252b376bba974e8696fc2p0L, 0x1.6390d4c6ad5476b5162f40e1d9a9p-88L, - 0x1.f50765b6e4540674f84b76p0L, 0x2.862baff99000dfc4352ba29b8908p-92L, - 0x1.f7bfdad9cbe138913b4bfep0L, 0x7.2bd95c5ce7280fa4d2344a3f5618p-92L, - 0x1.fa7c1819e90d82e90a7e74p0L, 0xb.263c1dc060c36f7650b4c0f233a8p-92L, - 0x1.fd3c22b8f71f10975ba4b2p0L, 0x1.2bcf3a5e12d269d8ad7c1a4a8875p-88L -}; - long double expl(long double x) { - union IEEEl2bits u, v; - long double q, r, r1, t, twopk, twopkp10000; - double dr, fn, r2; - int k, n, n2; + union IEEEl2bits u; + long double hi, lo, t, twopk; + int k; uint16_t hx, ix; + DOPRINT_START(&x); + /* Filter out exceptional cases. */ u.e = x; hx = u.xbits.expsign; @@ -241,60 +72,33 @@ expl(long double x) if (ix >= BIAS + 13) { /* |x| >= 8192 or x is NaN */ if (ix == BIAS + LDBL_MAX_EXP) { if (hx & 0x8000) /* x is -Inf or -NaN */ - return (-1 / x); - return (x + x); /* x is +Inf or +NaN */ + RETURNP(-1 / x); + RETURNP(x + x); /* x is +Inf or +NaN */ } if (x > o_threshold) - return (huge * huge); + RETURNP(huge * huge); if (x < u_threshold) - return (tiny * tiny); + RETURNP(tiny * tiny); } else if (ix < BIAS - 114) { /* |x| < 0x1p-114 */ - return (1 + x); /* 1 with inexact iff x != 0 */ + RETURN2P(1, x); /* 1 with inexact iff x != 0 */ } ENTERI(); - /* Reduce x to (k*ln2 + endpoint[n2] + r1 + r2). */ - /* Use a specialized rint() to get fn. Assume round-to-nearest. */ - /* XXX assume no extra precision for the additions, as for trig fns. */ - /* XXX this set of comments is now quadruplicated. */ - fn = (double)x * INV_L + 0x1.8p52 - 0x1.8p52; -#if defined(HAVE_EFFICIENT_IRINT) - n = irint(fn); -#else - n = (int)fn; -#endif - n2 = (unsigned)n % INTERVALS; - k = n >> LOG2_INTERVALS; - r1 = x - fn * L1; - r2 = fn * -L2; - r = r1 + r2; - - /* Prepare scale factors. */ - /* XXX sparc64 multiplication is so slow that scalbnl() is faster. */ - v.e = 1; - if (k >= LDBL_MIN_EXP) { - v.xbits.expsign = BIAS + k; - twopk = v.e; - } else { - v.xbits.expsign = BIAS + k + 10000; - twopkp10000 = v.e; - } - - /* Evaluate expl(endpoint[n2] + r1 + r2) = tbl[n2] * expl(r1 + r2). */ - dr = r; - q = r2 + r * r * (A2 + r * (A3 + r * (A4 + r * (A5 + r * (A6 + - dr * (A7 + dr * (A8 + dr * (A9 + dr * A10)))))))); - t = tbl[n2].lo + tbl[n2].hi; - t = tbl[n2].lo + t * (q + r1) + tbl[n2].hi; + twopk = 1; + __k_expl(x, &hi, &lo, &k); + t = SUM2P(hi, lo); /* Scale by 2**k. */ + /* XXX sparc64 multiplication is so slow that scalbnl() is faster. */ if (k >= LDBL_MIN_EXP) { if (k == LDBL_MAX_EXP) RETURNI(t * 2 * 0x1p16383L); + SET_LDBL_EXPSIGN(twopk, BIAS + k); RETURNI(t * twopk); } else { - RETURNI(t * twopkp10000 * twom10000); + SET_LDBL_EXPSIGN(twopk, BIAS + k + 10000); + RETURNI(t * twopk * twom10000); } } @@ -312,6 +116,12 @@ expl(long double x) * Setting T3 to 0 would require the |x| < 0x1p-113 condition to appear * in both subintervals, so set T3 = 2**-5, which places the condition * into the [T1, T3] interval. + * + * XXX we now do this more to (partially) balance the number of terms + * in the C and D polys than to avoid checking the condition in both + * intervals. + * + * XXX these micro-optimizations are excessive. */ static const double T1 = -0.1659, /* ~-30.625/128 * log(2) */ @@ -321,6 +131,12 @@ T3 = 0.03125; /* * Domain [-0.1659, 0.03125], range ~[2.9134e-44, 1.8404e-37]: * |(exp(x)-1-x-x**2/2)/x - p(x)| < 2**-122.03 + * + * XXX none of the long double C or D coeffs except C10 is correctly printed. + * If you re-print their values in %.35Le format, the result is always + * different. For example, the last 2 digits in C3 should be 59, not 67. + * 67 is apparently from rounding an extra-precision value to 36 decimal + * places. */ static const long double C3 = 1.66666666666666666666666666666666667e-1L, @@ -335,6 +151,13 @@ C11 = 2.50521083854417203619031960151253944e-8L, C12 = 2.08767569878679576457272282566520649e-9L, C13 = 1.60590438367252471783548748824255707e-10L; +/* + * XXX this has 1 more coeff than needed. + * XXX can start the double coeffs but not the double mults at C10. + * With my coeffs (C10-C17 double; s = best_s): + * Domain [-0.1659, 0.03125], range ~[-1.1976e-37, 1.1976e-37]: + * |(exp(x)-1-x-x**2/2)/x - p(x)| ~< 2**-122.65 + */ static const double C14 = 1.1470745580491932e-11, /* 0x1.93974a81dae30p-37 */ C15 = 7.6471620181090468e-13, /* 0x1.ae7f3820adab1p-41 */ @@ -359,6 +182,13 @@ D11 = 2.50521083855084570046480450935267433e-8L, D12 = 2.08767569819738524488686318024854942e-9L, D13 = 1.60590442297008495301927448122499313e-10L; +/* + * XXX this has 1 more coeff than needed. + * XXX can start the double coeffs but not the double mults at D11. + * With my coeffs (D11-D16 double): + * Domain [0.03125, 0.1659], range ~[-1.1980e-37, 1.1980e-37]: + * |(exp(x)-1-x-x**2/2)/x - p(x)| ~< 2**-122.65 + */ static const double D14 = 1.1470726176204336e-11, /* 0x1.93971dc395d9ep-37 */ D15 = 7.6478532249581686e-13, /* 0x1.ae892e3D16fcep-41 */ @@ -375,6 +205,8 @@ expm1l(long double x) int k, n, n2; uint16_t hx, ix; + DOPRINT_START(&x); + /* Filter out exceptional cases. */ u.e = x; hx = u.xbits.expsign; @@ -382,11 +214,11 @@ expm1l(long double x) if (ix >= BIAS + 7) { /* |x| >= 128 or x is NaN */ if (ix == BIAS + LDBL_MAX_EXP) { if (hx & 0x8000) /* x is -Inf or -NaN */ - return (-1 / x - 1); - return (x + x); /* x is +Inf or +NaN */ + RETURNP(-1 / x - 1); + RETURNP(x + x); /* x is +Inf or +NaN */ } if (x > o_threshold) - return (huge * huge); + RETURNP(huge * huge); /* * expm1l() never underflows, but it must avoid * unrepresentable large negative exponents. We used a @@ -395,7 +227,7 @@ expm1l(long double x) * in the same way as large ones here. */ if (hx & 0x8000) /* x <= -128 */ - return (tiny - 1); /* good for x < -114ln2 - eps */ + RETURN2P(tiny, -1); /* good for x < -114ln2 - eps */ } ENTERI(); @@ -407,7 +239,7 @@ expm1l(long double x) if (x < T3) { if (ix < BIAS - 113) { /* |x| < 0x1p-113 */ /* x (rounded) with inexact if x != 0: */ - RETURNI(x == 0 ? x : + RETURNPI(x == 0 ? x : (0x1p200 * x + fabsl(x)) * 0x1p-200); } q = x * x2 * C3 + x2 * x2 * (C4 + x * (C5 + x * (C6 + @@ -428,9 +260,9 @@ expm1l(long double x) hx2_hi = x_hi * x_hi / 2; hx2_lo = x_lo * (x + x_hi) / 2; if (ix >= BIAS - 7) - RETURNI(hx2_lo + x_lo + q + (hx2_hi + x_hi)); + RETURN2PI(hx2_hi + x_hi, hx2_lo + x_lo + q); else - RETURNI(hx2_lo + q + hx2_hi + x); + RETURN2PI(x, hx2_lo + q + hx2_hi); } /* Reduce x to (k*ln2 + endpoint[n2] + r1 + r2). */ @@ -463,21 +295,21 @@ expm1l(long double x) t = tbl[n2].lo + tbl[n2].hi; if (k == 0) { - t = tbl[n2].lo * (r1 + 1) + t * q + tbl[n2].hi * r1 + - (tbl[n2].hi - 1); + t = SUM2P(tbl[n2].hi - 1, tbl[n2].lo * (r1 + 1) + t * q + + tbl[n2].hi * r1); RETURNI(t); } if (k == -1) { - t = tbl[n2].lo * (r1 + 1) + t * q + tbl[n2].hi * r1 + - (tbl[n2].hi - 2); + t = SUM2P(tbl[n2].hi - 2, tbl[n2].lo * (r1 + 1) + t * q + + tbl[n2].hi * r1); RETURNI(t / 2); } if (k < -7) { - t = tbl[n2].lo + t * (q + r1) + tbl[n2].hi; + t = SUM2P(tbl[n2].hi, tbl[n2].lo + t * (q + r1)); RETURNI(t * twopk - 1); } if (k > 2 * LDBL_MANT_DIG - 1) { - t = tbl[n2].lo + t * (q + r1) + tbl[n2].hi; + t = SUM2P(tbl[n2].hi, tbl[n2].lo + t * (q + r1)); if (k == LDBL_MAX_EXP) RETURNI(t * 2 * 0x1p16383L - 1); RETURNI(t * twopk - 1); @@ -487,8 +319,8 @@ expm1l(long double x) twomk = v.e; if (k > LDBL_MANT_DIG - 1) - t = tbl[n2].lo - twomk + t * (q + r1) + tbl[n2].hi; + t = SUM2P(tbl[n2].hi, tbl[n2].lo - twomk + t * (q + r1)); else - t = tbl[n2].lo + t * (q + r1) + (tbl[n2].hi - twomk); + t = SUM2P(tbl[n2].hi - twomk, tbl[n2].lo + t * (q + r1)); RETURNI(t * twopk); } diff --git a/libm/upstream-freebsd/lib/msun/src/catrig.c b/libm/upstream-freebsd/lib/msun/src/catrig.c new file mode 100644 index 000000000..200977c0a --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/src/catrig.c @@ -0,0 +1,639 @@ +/*- + * Copyright (c) 2012 Stephen Montgomery-Smith + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include "math.h" +#include "math_private.h" + +#undef isinf +#define isinf(x) (fabs(x) == INFINITY) +#undef isnan +#define isnan(x) ((x) != (x)) +#define raise_inexact() do { volatile float junk = 1 + tiny; } while(0) +#undef signbit +#define signbit(x) (__builtin_signbit(x)) + +/* We need that DBL_EPSILON^2/128 is larger than FOUR_SQRT_MIN. */ +static const double +A_crossover = 10, /* Hull et al suggest 1.5, but 10 works better */ +B_crossover = 0.6417, /* suggested by Hull et al */ +FOUR_SQRT_MIN = 0x1p-509, /* >= 4 * sqrt(DBL_MIN) */ +QUARTER_SQRT_MAX = 0x1p509, /* <= sqrt(DBL_MAX) / 4 */ +m_e = 2.7182818284590452e0, /* 0x15bf0a8b145769.0p-51 */ +m_ln2 = 6.9314718055994531e-1, /* 0x162e42fefa39ef.0p-53 */ +pio2_hi = 1.5707963267948966e0, /* 0x1921fb54442d18.0p-52 */ +RECIP_EPSILON = 1 / DBL_EPSILON, +SQRT_3_EPSILON = 2.5809568279517849e-8, /* 0x1bb67ae8584caa.0p-78 */ +SQRT_6_EPSILON = 3.6500241499888571e-8, /* 0x13988e1409212e.0p-77 */ +SQRT_MIN = 0x1p-511; /* >= sqrt(DBL_MIN) */ + +static const volatile double +pio2_lo = 6.1232339957367659e-17; /* 0x11a62633145c07.0p-106 */ +static const volatile float +tiny = 0x1p-100; + +static double complex clog_for_large_values(double complex z); + +/* + * Testing indicates that all these functions are accurate up to 4 ULP. + * The functions casin(h) and cacos(h) are about 2.5 times slower than asinh. + * The functions catan(h) are a little under 2 times slower than atanh. + * + * The code for casinh, casin, cacos, and cacosh comes first. The code is + * rather complicated, and the four functions are highly interdependent. + * + * The code for catanh and catan comes at the end. It is much simpler than + * the other functions, and the code for these can be disconnected from the + * rest of the code. + */ + +/* + * ================================ + * | casinh, casin, cacos, cacosh | + * ================================ + */ + +/* + * The algorithm is very close to that in "Implementing the complex arcsine + * and arccosine functions using exception handling" by T. E. Hull, Thomas F. + * Fairgrieve, and Ping Tak Peter Tang, published in ACM Transactions on + * Mathematical Software, Volume 23 Issue 3, 1997, Pages 299-335, + * http://dl.acm.org/citation.cfm?id=275324. + * + * Throughout we use the convention z = x + I*y. + * + * casinh(z) = sign(x)*log(A+sqrt(A*A-1)) + I*asin(B) + * where + * A = (|z+I| + |z-I|) / 2 + * B = (|z+I| - |z-I|) / 2 = y/A + * + * These formulas become numerically unstable: + * (a) for Re(casinh(z)) when z is close to the line segment [-I, I] (that + * is, Re(casinh(z)) is close to 0); + * (b) for Im(casinh(z)) when z is close to either of the intervals + * [I, I*infinity) or (-I*infinity, -I] (that is, |Im(casinh(z))| is + * close to PI/2). + * + * These numerical problems are overcome by defining + * f(a, b) = (hypot(a, b) - b) / 2 = a*a / (hypot(a, b) + b) / 2 + * Then if A < A_crossover, we use + * log(A + sqrt(A*A-1)) = log1p((A-1) + sqrt((A-1)*(A+1))) + * A-1 = f(x, 1+y) + f(x, 1-y) + * and if B > B_crossover, we use + * asin(B) = atan2(y, sqrt(A*A - y*y)) = atan2(y, sqrt((A+y)*(A-y))) + * A-y = f(x, y+1) + f(x, y-1) + * where without loss of generality we have assumed that x and y are + * non-negative. + * + * Much of the difficulty comes because the intermediate computations may + * produce overflows or underflows. This is dealt with in the paper by Hull + * et al by using exception handling. We do this by detecting when + * computations risk underflow or overflow. The hardest part is handling the + * underflows when computing f(a, b). + * + * Note that the function f(a, b) does not appear explicitly in the paper by + * Hull et al, but the idea may be found on pages 308 and 309. Introducing the + * function f(a, b) allows us to concentrate many of the clever tricks in this + * paper into one function. + */ + +/* + * Function f(a, b, hypot_a_b) = (hypot(a, b) - b) / 2. + * Pass hypot(a, b) as the third argument. + */ +static inline double +f(double a, double b, double hypot_a_b) +{ + if (b < 0) + return ((hypot_a_b - b) / 2); + if (b == 0) + return (a / 2); + return (a * a / (hypot_a_b + b) / 2); +} + +/* + * All the hard work is contained in this function. + * x and y are assumed positive or zero, and less than RECIP_EPSILON. + * Upon return: + * rx = Re(casinh(z)) = -Im(cacos(y + I*x)). + * B_is_usable is set to 1 if the value of B is usable. + * If B_is_usable is set to 0, sqrt_A2my2 = sqrt(A*A - y*y), and new_y = y. + * If returning sqrt_A2my2 has potential to result in an underflow, it is + * rescaled, and new_y is similarly rescaled. + */ +static inline void +do_hard_work(double x, double y, double *rx, int *B_is_usable, double *B, + double *sqrt_A2my2, double *new_y) +{ + double R, S, A; /* A, B, R, and S are as in Hull et al. */ + double Am1, Amy; /* A-1, A-y. */ + + R = hypot(x, y + 1); /* |z+I| */ + S = hypot(x, y - 1); /* |z-I| */ + + /* A = (|z+I| + |z-I|) / 2 */ + A = (R + S) / 2; + /* + * Mathematically A >= 1. There is a small chance that this will not + * be so because of rounding errors. So we will make certain it is + * so. + */ + if (A < 1) + A = 1; + + if (A < A_crossover) { + /* + * Am1 = fp + fm, where fp = f(x, 1+y), and fm = f(x, 1-y). + * rx = log1p(Am1 + sqrt(Am1*(A+1))) + */ + if (y == 1 && x < DBL_EPSILON * DBL_EPSILON / 128) { + /* + * fp is of order x^2, and fm = x/2. + * A = 1 (inexactly). + */ + *rx = sqrt(x); + } else if (x >= DBL_EPSILON * fabs(y - 1)) { + /* + * Underflow will not occur because + * x >= DBL_EPSILON^2/128 >= FOUR_SQRT_MIN + */ + Am1 = f(x, 1 + y, R) + f(x, 1 - y, S); + *rx = log1p(Am1 + sqrt(Am1 * (A + 1))); + } else if (y < 1) { + /* + * fp = x*x/(1+y)/4, fm = x*x/(1-y)/4, and + * A = 1 (inexactly). + */ + *rx = x / sqrt((1 - y) * (1 + y)); + } else { /* if (y > 1) */ + /* + * A-1 = y-1 (inexactly). + */ + *rx = log1p((y - 1) + sqrt((y - 1) * (y + 1))); + } + } else { + *rx = log(A + sqrt(A * A - 1)); + } + + *new_y = y; + + if (y < FOUR_SQRT_MIN) { + /* + * Avoid a possible underflow caused by y/A. For casinh this + * would be legitimate, but will be picked up by invoking atan2 + * later on. For cacos this would not be legitimate. + */ + *B_is_usable = 0; + *sqrt_A2my2 = A * (2 / DBL_EPSILON); + *new_y = y * (2 / DBL_EPSILON); + return; + } + + /* B = (|z+I| - |z-I|) / 2 = y/A */ + *B = y / A; + *B_is_usable = 1; + + if (*B > B_crossover) { + *B_is_usable = 0; + /* + * Amy = fp + fm, where fp = f(x, y+1), and fm = f(x, y-1). + * sqrt_A2my2 = sqrt(Amy*(A+y)) + */ + if (y == 1 && x < DBL_EPSILON / 128) { + /* + * fp is of order x^2, and fm = x/2. + * A = 1 (inexactly). + */ + *sqrt_A2my2 = sqrt(x) * sqrt((A + y) / 2); + } else if (x >= DBL_EPSILON * fabs(y - 1)) { + /* + * Underflow will not occur because + * x >= DBL_EPSILON/128 >= FOUR_SQRT_MIN + * and + * x >= DBL_EPSILON^2 >= FOUR_SQRT_MIN + */ + Amy = f(x, y + 1, R) + f(x, y - 1, S); + *sqrt_A2my2 = sqrt(Amy * (A + y)); + } else if (y > 1) { + /* + * fp = x*x/(y+1)/4, fm = x*x/(y-1)/4, and + * A = y (inexactly). + * + * y < RECIP_EPSILON. So the following + * scaling should avoid any underflow problems. + */ + *sqrt_A2my2 = x * (4 / DBL_EPSILON / DBL_EPSILON) * y / + sqrt((y + 1) * (y - 1)); + *new_y = y * (4 / DBL_EPSILON / DBL_EPSILON); + } else { /* if (y < 1) */ + /* + * fm = 1-y >= DBL_EPSILON, fp is of order x^2, and + * A = 1 (inexactly). + */ + *sqrt_A2my2 = sqrt((1 - y) * (1 + y)); + } + } +} + +/* + * casinh(z) = z + O(z^3) as z -> 0 + * + * casinh(z) = sign(x)*clog(sign(x)*z) + O(1/z^2) as z -> infinity + * The above formula works for the imaginary part as well, because + * Im(casinh(z)) = sign(x)*atan2(sign(x)*y, fabs(x)) + O(y/z^3) + * as z -> infinity, uniformly in y + */ +double complex +casinh(double complex z) +{ + double x, y, ax, ay, rx, ry, B, sqrt_A2my2, new_y; + int B_is_usable; + double complex w; + + x = creal(z); + y = cimag(z); + ax = fabs(x); + ay = fabs(y); + + if (isnan(x) || isnan(y)) { + /* casinh(+-Inf + I*NaN) = +-Inf + I*NaN */ + if (isinf(x)) + return (cpack(x, y + y)); + /* casinh(NaN + I*+-Inf) = opt(+-)Inf + I*NaN */ + if (isinf(y)) + return (cpack(y, x + x)); + /* casinh(NaN + I*0) = NaN + I*0 */ + if (y == 0) + return (cpack(x + x, y)); + /* + * All other cases involving NaN return NaN + I*NaN. + * C99 leaves it optional whether to raise invalid if one of + * the arguments is not NaN, so we opt not to raise it. + */ + return (cpack(x + 0.0L + (y + 0), x + 0.0L + (y + 0))); + } + + if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) { + /* clog...() will raise inexact unless x or y is infinite. */ + if (signbit(x) == 0) + w = clog_for_large_values(z) + m_ln2; + else + w = clog_for_large_values(-z) + m_ln2; + return (cpack(copysign(creal(w), x), copysign(cimag(w), y))); + } + + /* Avoid spuriously raising inexact for z = 0. */ + if (x == 0 && y == 0) + return (z); + + /* All remaining cases are inexact. */ + raise_inexact(); + + if (ax < SQRT_6_EPSILON / 4 && ay < SQRT_6_EPSILON / 4) + return (z); + + do_hard_work(ax, ay, &rx, &B_is_usable, &B, &sqrt_A2my2, &new_y); + if (B_is_usable) + ry = asin(B); + else + ry = atan2(new_y, sqrt_A2my2); + return (cpack(copysign(rx, x), copysign(ry, y))); +} + +/* + * casin(z) = reverse(casinh(reverse(z))) + * where reverse(x + I*y) = y + I*x = I*conj(z). + */ +double complex +casin(double complex z) +{ + double complex w = casinh(cpack(cimag(z), creal(z))); + + return (cpack(cimag(w), creal(w))); +} + +/* + * cacos(z) = PI/2 - casin(z) + * but do the computation carefully so cacos(z) is accurate when z is + * close to 1. + * + * cacos(z) = PI/2 - z + O(z^3) as z -> 0 + * + * cacos(z) = -sign(y)*I*clog(z) + O(1/z^2) as z -> infinity + * The above formula works for the real part as well, because + * Re(cacos(z)) = atan2(fabs(y), x) + O(y/z^3) + * as z -> infinity, uniformly in y + */ +double complex +cacos(double complex z) +{ + double x, y, ax, ay, rx, ry, B, sqrt_A2mx2, new_x; + int sx, sy; + int B_is_usable; + double complex w; + + x = creal(z); + y = cimag(z); + sx = signbit(x); + sy = signbit(y); + ax = fabs(x); + ay = fabs(y); + + if (isnan(x) || isnan(y)) { + /* cacos(+-Inf + I*NaN) = NaN + I*opt(-)Inf */ + if (isinf(x)) + return (cpack(y + y, -INFINITY)); + /* cacos(NaN + I*+-Inf) = NaN + I*-+Inf */ + if (isinf(y)) + return (cpack(x + x, -y)); + /* cacos(0 + I*NaN) = PI/2 + I*NaN with inexact */ + if (x == 0) + return (cpack(pio2_hi + pio2_lo, y + y)); + /* + * All other cases involving NaN return NaN + I*NaN. + * C99 leaves it optional whether to raise invalid if one of + * the arguments is not NaN, so we opt not to raise it. + */ + return (cpack(x + 0.0L + (y + 0), x + 0.0L + (y + 0))); + } + + if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) { + /* clog...() will raise inexact unless x or y is infinite. */ + w = clog_for_large_values(z); + rx = fabs(cimag(w)); + ry = creal(w) + m_ln2; + if (sy == 0) + ry = -ry; + return (cpack(rx, ry)); + } + + /* Avoid spuriously raising inexact for z = 1. */ + if (x == 1 && y == 0) + return (cpack(0, -y)); + + /* All remaining cases are inexact. */ + raise_inexact(); + + if (ax < SQRT_6_EPSILON / 4 && ay < SQRT_6_EPSILON / 4) + return (cpack(pio2_hi - (x - pio2_lo), -y)); + + do_hard_work(ay, ax, &ry, &B_is_usable, &B, &sqrt_A2mx2, &new_x); + if (B_is_usable) { + if (sx == 0) + rx = acos(B); + else + rx = acos(-B); + } else { + if (sx == 0) + rx = atan2(sqrt_A2mx2, new_x); + else + rx = atan2(sqrt_A2mx2, -new_x); + } + if (sy == 0) + ry = -ry; + return (cpack(rx, ry)); +} + +/* + * cacosh(z) = I*cacos(z) or -I*cacos(z) + * where the sign is chosen so Re(cacosh(z)) >= 0. + */ +double complex +cacosh(double complex z) +{ + double complex w; + double rx, ry; + + w = cacos(z); + rx = creal(w); + ry = cimag(w); + /* cacosh(NaN + I*NaN) = NaN + I*NaN */ + if (isnan(rx) && isnan(ry)) + return (cpack(ry, rx)); + /* cacosh(NaN + I*+-Inf) = +Inf + I*NaN */ + /* cacosh(+-Inf + I*NaN) = +Inf + I*NaN */ + if (isnan(rx)) + return (cpack(fabs(ry), rx)); + /* cacosh(0 + I*NaN) = NaN + I*NaN */ + if (isnan(ry)) + return (cpack(ry, ry)); + return (cpack(fabs(ry), copysign(rx, cimag(z)))); +} + +/* + * Optimized version of clog() for |z| finite and larger than ~RECIP_EPSILON. + */ +static double complex +clog_for_large_values(double complex z) +{ + double x, y; + double ax, ay, t; + + x = creal(z); + y = cimag(z); + ax = fabs(x); + ay = fabs(y); + if (ax < ay) { + t = ax; + ax = ay; + ay = t; + } + + /* + * Avoid overflow in hypot() when x and y are both very large. + * Divide x and y by E, and then add 1 to the logarithm. This depends + * on E being larger than sqrt(2). + * Dividing by E causes an insignificant loss of accuracy; however + * this method is still poor since it is uneccessarily slow. + */ + if (ax > DBL_MAX / 2) + return (cpack(log(hypot(x / m_e, y / m_e)) + 1, atan2(y, x))); + + /* + * Avoid overflow when x or y is large. Avoid underflow when x or + * y is small. + */ + if (ax > QUARTER_SQRT_MAX || ay < SQRT_MIN) + return (cpack(log(hypot(x, y)), atan2(y, x))); + + return (cpack(log(ax * ax + ay * ay) / 2, atan2(y, x))); +} + +/* + * ================= + * | catanh, catan | + * ================= + */ + +/* + * sum_squares(x,y) = x*x + y*y (or just x*x if y*y would underflow). + * Assumes x*x and y*y will not overflow. + * Assumes x and y are finite. + * Assumes y is non-negative. + * Assumes fabs(x) >= DBL_EPSILON. + */ +static inline double +sum_squares(double x, double y) +{ + + /* Avoid underflow when y is small. */ + if (y < SQRT_MIN) + return (x * x); + + return (x * x + y * y); +} + +/* + * real_part_reciprocal(x, y) = Re(1/(x+I*y)) = x/(x*x + y*y). + * Assumes x and y are not NaN, and one of x and y is larger than + * RECIP_EPSILON. We avoid unwarranted underflow. It is important to not use + * the code creal(1/z), because the imaginary part may produce an unwanted + * underflow. + * This is only called in a context where inexact is always raised before + * the call, so no effort is made to avoid or force inexact. + */ +static inline double +real_part_reciprocal(double x, double y) +{ + double scale; + uint32_t hx, hy; + int32_t ix, iy; + + /* + * This code is inspired by the C99 document n1124.pdf, Section G.5.1, + * example 2. + */ + GET_HIGH_WORD(hx, x); + ix = hx & 0x7ff00000; + GET_HIGH_WORD(hy, y); + iy = hy & 0x7ff00000; +#define BIAS (DBL_MAX_EXP - 1) +/* XXX more guard digits are useful iff there is extra precision. */ +#define CUTOFF (DBL_MANT_DIG / 2 + 1) /* just half or 1 guard digit */ + if (ix - iy >= CUTOFF << 20 || isinf(x)) + return (1 / x); /* +-Inf -> +-0 is special */ + if (iy - ix >= CUTOFF << 20) + return (x / y / y); /* should avoid double div, but hard */ + if (ix <= (BIAS + DBL_MAX_EXP / 2 - CUTOFF) << 20) + return (x / (x * x + y * y)); + scale = 1; + SET_HIGH_WORD(scale, 0x7ff00000 - ix); /* 2**(1-ilogb(x)) */ + x *= scale; + y *= scale; + return (x / (x * x + y * y) * scale); +} + +/* + * catanh(z) = log((1+z)/(1-z)) / 2 + * = log1p(4*x / |z-1|^2) / 4 + * + I * atan2(2*y, (1-x)*(1+x)-y*y) / 2 + * + * catanh(z) = z + O(z^3) as z -> 0 + * + * catanh(z) = 1/z + sign(y)*I*PI/2 + O(1/z^3) as z -> infinity + * The above formula works for the real part as well, because + * Re(catanh(z)) = x/|z|^2 + O(x/z^4) + * as z -> infinity, uniformly in x + */ +double complex +catanh(double complex z) +{ + double x, y, ax, ay, rx, ry; + + x = creal(z); + y = cimag(z); + ax = fabs(x); + ay = fabs(y); + + /* This helps handle many cases. */ + if (y == 0 && ax <= 1) + return (cpack(atanh(x), y)); + + /* To ensure the same accuracy as atan(), and to filter out z = 0. */ + if (x == 0) + return (cpack(x, atan(y))); + + if (isnan(x) || isnan(y)) { + /* catanh(+-Inf + I*NaN) = +-0 + I*NaN */ + if (isinf(x)) + return (cpack(copysign(0, x), y + y)); + /* catanh(NaN + I*+-Inf) = sign(NaN)0 + I*+-PI/2 */ + if (isinf(y)) + return (cpack(copysign(0, x), + copysign(pio2_hi + pio2_lo, y))); + /* + * All other cases involving NaN return NaN + I*NaN. + * C99 leaves it optional whether to raise invalid if one of + * the arguments is not NaN, so we opt not to raise it. + */ + return (cpack(x + 0.0L + (y + 0), x + 0.0L + (y + 0))); + } + + if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) + return (cpack(real_part_reciprocal(x, y), + copysign(pio2_hi + pio2_lo, y))); + + if (ax < SQRT_3_EPSILON / 2 && ay < SQRT_3_EPSILON / 2) { + /* + * z = 0 was filtered out above. All other cases must raise + * inexact, but this is the only only that needs to do it + * explicitly. + */ + raise_inexact(); + return (z); + } + + if (ax == 1 && ay < DBL_EPSILON) + rx = (m_ln2 - log(ay)) / 2; + else + rx = log1p(4 * ax / sum_squares(ax - 1, ay)) / 4; + + if (ax == 1) + ry = atan2(2, -ay) / 2; + else if (ay < DBL_EPSILON) + ry = atan2(2 * ay, (1 - ax) * (1 + ax)) / 2; + else + ry = atan2(2 * ay, (1 - ax) * (1 + ax) - ay * ay) / 2; + + return (cpack(copysign(rx, x), copysign(ry, y))); +} + +/* + * catan(z) = reverse(catanh(reverse(z))) + * where reverse(x + I*y) = y + I*x = I*conj(z). + */ +double complex +catan(double complex z) +{ + double complex w = catanh(cpack(cimag(z), creal(z))); + + return (cpack(cimag(w), creal(w))); +} diff --git a/libm/upstream-freebsd/lib/msun/src/catrigf.c b/libm/upstream-freebsd/lib/msun/src/catrigf.c new file mode 100644 index 000000000..08ebef783 --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/src/catrigf.c @@ -0,0 +1,393 @@ +/*- + * Copyright (c) 2012 Stephen Montgomery-Smith + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * The algorithm is very close to that in "Implementing the complex arcsine + * and arccosine functions using exception handling" by T. E. Hull, Thomas F. + * Fairgrieve, and Ping Tak Peter Tang, published in ACM Transactions on + * Mathematical Software, Volume 23 Issue 3, 1997, Pages 299-335, + * http://dl.acm.org/citation.cfm?id=275324. + * + * See catrig.c for complete comments. + * + * XXX comments were removed automatically, and even short ones on the right + * of statements were removed (all of them), contrary to normal style. Only + * a few comments on the right of declarations remain. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include "math.h" +#include "math_private.h" + +#undef isinf +#define isinf(x) (fabsf(x) == INFINITY) +#undef isnan +#define isnan(x) ((x) != (x)) +#define raise_inexact() do { volatile float junk = 1 + tiny; } while(0) +#undef signbit +#define signbit(x) (__builtin_signbitf(x)) + +static const float +A_crossover = 10, +B_crossover = 0.6417, +FOUR_SQRT_MIN = 0x1p-61, +QUARTER_SQRT_MAX = 0x1p61, +m_e = 2.7182818285e0, /* 0xadf854.0p-22 */ +m_ln2 = 6.9314718056e-1, /* 0xb17218.0p-24 */ +pio2_hi = 1.5707962513e0, /* 0xc90fda.0p-23 */ +RECIP_EPSILON = 1 / FLT_EPSILON, +SQRT_3_EPSILON = 5.9801995673e-4, /* 0x9cc471.0p-34 */ +SQRT_6_EPSILON = 8.4572793338e-4, /* 0xddb3d7.0p-34 */ +SQRT_MIN = 0x1p-63; + +static const volatile float +pio2_lo = 7.5497899549e-8, /* 0xa22169.0p-47 */ +tiny = 0x1p-100; + +static float complex clog_for_large_values(float complex z); + +static inline float +f(float a, float b, float hypot_a_b) +{ + if (b < 0) + return ((hypot_a_b - b) / 2); + if (b == 0) + return (a / 2); + return (a * a / (hypot_a_b + b) / 2); +} + +static inline void +do_hard_work(float x, float y, float *rx, int *B_is_usable, float *B, + float *sqrt_A2my2, float *new_y) +{ + float R, S, A; + float Am1, Amy; + + R = hypotf(x, y + 1); + S = hypotf(x, y - 1); + + A = (R + S) / 2; + if (A < 1) + A = 1; + + if (A < A_crossover) { + if (y == 1 && x < FLT_EPSILON * FLT_EPSILON / 128) { + *rx = sqrtf(x); + } else if (x >= FLT_EPSILON * fabsf(y - 1)) { + Am1 = f(x, 1 + y, R) + f(x, 1 - y, S); + *rx = log1pf(Am1 + sqrtf(Am1 * (A + 1))); + } else if (y < 1) { + *rx = x / sqrtf((1 - y) * (1 + y)); + } else { + *rx = log1pf((y - 1) + sqrtf((y - 1) * (y + 1))); + } + } else { + *rx = logf(A + sqrtf(A * A - 1)); + } + + *new_y = y; + + if (y < FOUR_SQRT_MIN) { + *B_is_usable = 0; + *sqrt_A2my2 = A * (2 / FLT_EPSILON); + *new_y = y * (2 / FLT_EPSILON); + return; + } + + *B = y / A; + *B_is_usable = 1; + + if (*B > B_crossover) { + *B_is_usable = 0; + if (y == 1 && x < FLT_EPSILON / 128) { + *sqrt_A2my2 = sqrtf(x) * sqrtf((A + y) / 2); + } else if (x >= FLT_EPSILON * fabsf(y - 1)) { + Amy = f(x, y + 1, R) + f(x, y - 1, S); + *sqrt_A2my2 = sqrtf(Amy * (A + y)); + } else if (y > 1) { + *sqrt_A2my2 = x * (4 / FLT_EPSILON / FLT_EPSILON) * y / + sqrtf((y + 1) * (y - 1)); + *new_y = y * (4 / FLT_EPSILON / FLT_EPSILON); + } else { + *sqrt_A2my2 = sqrtf((1 - y) * (1 + y)); + } + } +} + +float complex +casinhf(float complex z) +{ + float x, y, ax, ay, rx, ry, B, sqrt_A2my2, new_y; + int B_is_usable; + float complex w; + + x = crealf(z); + y = cimagf(z); + ax = fabsf(x); + ay = fabsf(y); + + if (isnan(x) || isnan(y)) { + if (isinf(x)) + return (cpackf(x, y + y)); + if (isinf(y)) + return (cpackf(y, x + x)); + if (y == 0) + return (cpackf(x + x, y)); + return (cpackf(x + 0.0L + (y + 0), x + 0.0L + (y + 0))); + } + + if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) { + if (signbit(x) == 0) + w = clog_for_large_values(z) + m_ln2; + else + w = clog_for_large_values(-z) + m_ln2; + return (cpackf(copysignf(crealf(w), x), + copysignf(cimagf(w), y))); + } + + if (x == 0 && y == 0) + return (z); + + raise_inexact(); + + if (ax < SQRT_6_EPSILON / 4 && ay < SQRT_6_EPSILON / 4) + return (z); + + do_hard_work(ax, ay, &rx, &B_is_usable, &B, &sqrt_A2my2, &new_y); + if (B_is_usable) + ry = asinf(B); + else + ry = atan2f(new_y, sqrt_A2my2); + return (cpackf(copysignf(rx, x), copysignf(ry, y))); +} + +float complex +casinf(float complex z) +{ + float complex w = casinhf(cpackf(cimagf(z), crealf(z))); + + return (cpackf(cimagf(w), crealf(w))); +} + +float complex +cacosf(float complex z) +{ + float x, y, ax, ay, rx, ry, B, sqrt_A2mx2, new_x; + int sx, sy; + int B_is_usable; + float complex w; + + x = crealf(z); + y = cimagf(z); + sx = signbit(x); + sy = signbit(y); + ax = fabsf(x); + ay = fabsf(y); + + if (isnan(x) || isnan(y)) { + if (isinf(x)) + return (cpackf(y + y, -INFINITY)); + if (isinf(y)) + return (cpackf(x + x, -y)); + if (x == 0) + return (cpackf(pio2_hi + pio2_lo, y + y)); + return (cpackf(x + 0.0L + (y + 0), x + 0.0L + (y + 0))); + } + + if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) { + w = clog_for_large_values(z); + rx = fabsf(cimagf(w)); + ry = crealf(w) + m_ln2; + if (sy == 0) + ry = -ry; + return (cpackf(rx, ry)); + } + + if (x == 1 && y == 0) + return (cpackf(0, -y)); + + raise_inexact(); + + if (ax < SQRT_6_EPSILON / 4 && ay < SQRT_6_EPSILON / 4) + return (cpackf(pio2_hi - (x - pio2_lo), -y)); + + do_hard_work(ay, ax, &ry, &B_is_usable, &B, &sqrt_A2mx2, &new_x); + if (B_is_usable) { + if (sx == 0) + rx = acosf(B); + else + rx = acosf(-B); + } else { + if (sx == 0) + rx = atan2f(sqrt_A2mx2, new_x); + else + rx = atan2f(sqrt_A2mx2, -new_x); + } + if (sy == 0) + ry = -ry; + return (cpackf(rx, ry)); +} + +float complex +cacoshf(float complex z) +{ + float complex w; + float rx, ry; + + w = cacosf(z); + rx = crealf(w); + ry = cimagf(w); + if (isnan(rx) && isnan(ry)) + return (cpackf(ry, rx)); + if (isnan(rx)) + return (cpackf(fabsf(ry), rx)); + if (isnan(ry)) + return (cpackf(ry, ry)); + return (cpackf(fabsf(ry), copysignf(rx, cimagf(z)))); +} + +static float complex +clog_for_large_values(float complex z) +{ + float x, y; + float ax, ay, t; + + x = crealf(z); + y = cimagf(z); + ax = fabsf(x); + ay = fabsf(y); + if (ax < ay) { + t = ax; + ax = ay; + ay = t; + } + + if (ax > FLT_MAX / 2) + return (cpackf(logf(hypotf(x / m_e, y / m_e)) + 1, + atan2f(y, x))); + + if (ax > QUARTER_SQRT_MAX || ay < SQRT_MIN) + return (cpackf(logf(hypotf(x, y)), atan2f(y, x))); + + return (cpackf(logf(ax * ax + ay * ay) / 2, atan2f(y, x))); +} + +static inline float +sum_squares(float x, float y) +{ + + if (y < SQRT_MIN) + return (x * x); + + return (x * x + y * y); +} + +static inline float +real_part_reciprocal(float x, float y) +{ + float scale; + uint32_t hx, hy; + int32_t ix, iy; + + GET_FLOAT_WORD(hx, x); + ix = hx & 0x7f800000; + GET_FLOAT_WORD(hy, y); + iy = hy & 0x7f800000; +#define BIAS (FLT_MAX_EXP - 1) +#define CUTOFF (FLT_MANT_DIG / 2 + 1) + if (ix - iy >= CUTOFF << 23 || isinf(x)) + return (1 / x); + if (iy - ix >= CUTOFF << 23) + return (x / y / y); + if (ix <= (BIAS + FLT_MAX_EXP / 2 - CUTOFF) << 23) + return (x / (x * x + y * y)); + SET_FLOAT_WORD(scale, 0x7f800000 - ix); + x *= scale; + y *= scale; + return (x / (x * x + y * y) * scale); +} + +float complex +catanhf(float complex z) +{ + float x, y, ax, ay, rx, ry; + + x = crealf(z); + y = cimagf(z); + ax = fabsf(x); + ay = fabsf(y); + + if (y == 0 && ax <= 1) + return (cpackf(atanhf(x), y)); + + if (x == 0) + return (cpackf(x, atanf(y))); + + if (isnan(x) || isnan(y)) { + if (isinf(x)) + return (cpackf(copysignf(0, x), y + y)); + if (isinf(y)) + return (cpackf(copysignf(0, x), + copysignf(pio2_hi + pio2_lo, y))); + return (cpackf(x + 0.0L + (y + 0), x + 0.0L + (y + 0))); + } + + if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) + return (cpackf(real_part_reciprocal(x, y), + copysignf(pio2_hi + pio2_lo, y))); + + if (ax < SQRT_3_EPSILON / 2 && ay < SQRT_3_EPSILON / 2) { + raise_inexact(); + return (z); + } + + if (ax == 1 && ay < FLT_EPSILON) + rx = (m_ln2 - logf(ay)) / 2; + else + rx = log1pf(4 * ax / sum_squares(ax - 1, ay)) / 4; + + if (ax == 1) + ry = atan2f(2, -ay) / 2; + else if (ay < FLT_EPSILON) + ry = atan2f(2 * ay, (1 - ax) * (1 + ax)) / 2; + else + ry = atan2f(2 * ay, (1 - ax) * (1 + ax) - ay * ay) / 2; + + return (cpackf(copysignf(rx, x), copysignf(ry, y))); +} + +float complex +catanf(float complex z) +{ + float complex w = catanhf(cpackf(cimagf(z), crealf(z))); + + return (cpackf(cimagf(w), crealf(w))); +} diff --git a/libm/upstream-freebsd/lib/msun/src/e_cosh.c b/libm/upstream-freebsd/lib/msun/src/e_cosh.c index a36369545..246b5fbec 100644 --- a/libm/upstream-freebsd/lib/msun/src/e_cosh.c +++ b/libm/upstream-freebsd/lib/msun/src/e_cosh.c @@ -35,6 +35,8 @@ __FBSDID("$FreeBSD$"); * only cosh(0)=1 is exact for finite x. */ +#include + #include "math.h" #include "math_private.h" @@ -77,3 +79,7 @@ __ieee754_cosh(double x) /* |x| > overflowthresold, cosh(x) overflow */ return huge*huge; } + +#if (LDBL_MANT_DIG == 53) +__weak_reference(cosh, coshl); +#endif diff --git a/libm/upstream-freebsd/lib/msun/src/e_coshl.c b/libm/upstream-freebsd/lib/msun/src/e_coshl.c new file mode 100644 index 000000000..0a21277fc --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/src/e_coshl.c @@ -0,0 +1,130 @@ +/* from: FreeBSD: head/lib/msun/src/e_coshl.c XXX */ + +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +#include +__FBSDID("$FreeBSD$"); + +/* + * See e_cosh.c for complete comments. + * + * Converted to long double by Bruce D. Evans. + */ + +#include +#ifdef __i386__ +#include +#endif + +#include "fpmath.h" +#include "math.h" +#include "math_private.h" +#include "k_expl.h" + +#if LDBL_MAX_EXP != 0x4000 +/* We also require the usual expsign encoding. */ +#error "Unsupported long double format" +#endif + +#define BIAS (LDBL_MAX_EXP - 1) + +static const volatile long double huge = 0x1p10000L, tiny = 0x1p-10000L; +#if LDBL_MANT_DIG == 64 +/* + * Domain [-1, 1], range ~[-1.8211e-21, 1.8211e-21]: + * |cosh(x) - c(x)| < 2**-68.8 + */ +static const union IEEEl2bits +C4u = LD80C(0xaaaaaaaaaaaaac78, -5, 4.16666666666666682297e-2L); +#define C4 C4u.e +static const double +C2 = 0.5, +C6 = 1.3888888888888616e-3, /* 0x16c16c16c16b99.0p-62 */ +C8 = 2.4801587301767953e-5, /* 0x1a01a01a027061.0p-68 */ +C10 = 2.7557319163300398e-7, /* 0x127e4fb6c9b55f.0p-74 */ +C12 = 2.0876768371393075e-9, /* 0x11eed99406a3f4.0p-81 */ +C14 = 1.1469537039374480e-11, /* 0x1938c67cd18c48.0p-89 */ +C16 = 4.8473490896852041e-14; /* 0x1b49c429701e45.0p-97 */ +#elif LDBL_MANT_DIG == 113 +/* + * Domain [-1, 1], range ~[-2.3194e-37, 2.3194e-37]: + * |cosh(x) - c(x)| < 2**-121.69 + */ +static const long double +C4 = 4.16666666666666666666666666666666225e-2L, /* 0x1555555555555555555555555554e.0p-117L */ +C6 = 1.38888888888888888888888888889434831e-3L, /* 0x16c16c16c16c16c16c16c16c1dd7a.0p-122L */ +C8 = 2.48015873015873015873015871870962089e-5L, /* 0x1a01a01a01a01a01a01a017af2756.0p-128L */ +C10 = 2.75573192239858906525574318600800201e-7L, /* 0x127e4fb7789f5c72ef01c8a040640.0p-134L */ +C12 = 2.08767569878680989791444691755468269e-9L, /* 0x11eed8eff8d897b543d0679607399.0p-141L */ +C14= 1.14707455977297247387801189650495351e-11L, /* 0x193974a8c07c9d24ae169a7fa9b54.0p-149L */ +C16 = 4.77947733238737883626416876486279985e-14L; /* 0x1ae7f3e733b814d4e1b90f5727fe4.0p-157L */ +static const double +C2 = 0.5, +C18 = 1.5619206968597871e-16, /* 0x16827863b9900b.0p-105 */ +C20 = 4.1103176218528049e-19, /* 0x1e542ba3d3c269.0p-114 */ +C22 = 8.8967926401641701e-22, /* 0x10ce399542a014.0p-122 */ +C24 = 1.6116681626523904e-24, /* 0x1f2c981d1f0cb7.0p-132 */ +C26 = 2.5022374732804632e-27; /* 0x18c7ecf8b2c4a0.0p-141 */ +#else +#error "Unsupported long double format" +#endif /* LDBL_MANT_DIG == 64 */ + +/* log(2**16385 - 0.5) rounded up: */ +static const float +o_threshold = 1.13572168e4; /* 0xb174de.0p-10 */ + +long double +coshl(long double x) +{ + long double hi,lo,x2,x4; + double dx2; + uint16_t ix; + + GET_LDBL_EXPSIGN(ix,x); + ix &= 0x7fff; + + /* x is INF or NaN */ + if(ix>=0x7fff) return x*x; + + ENTERI(); + + /* |x| < 1, return 1 or c(x) */ + if(ix<0x3fff) { + if (ix o_threshold, cosh(x) overflow */ + RETURNI(huge*huge); +} diff --git a/libm/upstream-freebsd/lib/msun/src/e_lgamma.c b/libm/upstream-freebsd/lib/msun/src/e_lgamma.c index 4674d9bf0..43f517560 100644 --- a/libm/upstream-freebsd/lib/msun/src/e_lgamma.c +++ b/libm/upstream-freebsd/lib/msun/src/e_lgamma.c @@ -21,6 +21,8 @@ __FBSDID("$FreeBSD$"); * Method: call __ieee754_lgamma_r */ +#include + #include "math.h" #include "math_private.h" @@ -31,3 +33,7 @@ __ieee754_lgamma(double x) { return __ieee754_lgamma_r(x,&signgam); } + +#if (LDBL_MANT_DIG == 53) +__weak_reference(lgamma, lgammal); +#endif diff --git a/libm/upstream-freebsd/lib/msun/src/e_lgamma_r.c b/libm/upstream-freebsd/lib/msun/src/e_lgamma_r.c index 1cff592c5..a6fd691ff 100644 --- a/libm/upstream-freebsd/lib/msun/src/e_lgamma_r.c +++ b/libm/upstream-freebsd/lib/msun/src/e_lgamma_r.c @@ -83,11 +83,15 @@ __FBSDID("$FreeBSD$"); * */ +#include + #include "math.h" #include "math_private.h" +static const volatile double vzero = 0; + static const double -two52= 4.50359962737049600000e+15, /* 0x43300000, 0x00000000 */ +zero= 0.00000000000000000000e+00, half= 5.00000000000000000000e-01, /* 0x3FE00000, 0x00000000 */ one = 1.00000000000000000000e+00, /* 0x3FF00000, 0x00000000 */ pi = 3.14159265358979311600e+00, /* 0x400921FB, 0x54442D18 */ @@ -154,39 +158,35 @@ w4 = -5.95187557450339963135e-04, /* 0xBF4380CB, 0x8C0FE741 */ w5 = 8.36339918996282139126e-04, /* 0x3F4B67BA, 0x4CDAD5D1 */ w6 = -1.63092934096575273989e-03; /* 0xBF5AB89D, 0x0B9E43E4 */ -static const double zero= 0.00000000000000000000e+00; - - static double sin_pi(double x) +/* + * Compute sin(pi*x) without actually doing the pi*x multiplication. + * sin_pi(x) is only called for x < 0 and |x| < 2**(p-1) where p is + * the precision of x. + */ +static double +sin_pi(double x) { + volatile double vz; double y,z; - int n,ix; + int n; - GET_HIGH_WORD(ix,x); - ix &= 0x7fffffff; + y = -x; - if(ix<0x3fd00000) return __kernel_sin(pi*x,zero,0); - y = -x; /* x is assume negative */ + vz = y+0x1p52; /* depend on 0 <= y < 0x1p52 */ + z = vz-0x1p52; /* rint(y) for the above range */ + if (z == y) + return zero; + + vz = y+0x1p50; + GET_LOW_WORD(n,vz); /* bits for rounded y (units 0.25) */ + z = vz-0x1p50; /* y rounded to a multiple of 0.25 */ + if (z > y) { + z -= 0.25; /* adjust to round down */ + n--; + } + n &= 7; /* octant of y mod 2 */ + y = y - z + n * 0.25; /* y mod 2 */ - /* - * argument reduction, make sure inexact flag not raised if input - * is an integer - */ - z = floor(y); - if(z!=y) { /* inexact anyway */ - y *= 0.5; - y = 2.0*(y - floor(y)); /* y = |x| mod 2.0 */ - n = (int) (y*4.0); - } else { - if(ix>=0x43400000) { - y = zero; n = 0; /* y must be even */ - } else { - if(ix<0x43300000) z = y+two52; /* exact */ - GET_LOW_WORD(n,z); - n &= 1; - y = n; - n<<= 2; - } - } switch (n) { case 0: y = __kernel_sin(pi*y,zero,0); break; case 1: @@ -206,7 +206,7 @@ __ieee754_lgamma_r(double x, int *signgamp) { double t,y,z,nadj,p,p1,p2,p3,q,r,w; int32_t hx; - int i,lx,ix; + int i,ix,lx; EXTRACT_WORDS(hx,lx,x); @@ -214,7 +214,11 @@ __ieee754_lgamma_r(double x, int *signgamp) *signgamp = 1; ix = hx&0x7fffffff; if(ix>=0x7ff00000) return x*x; - if((ix|lx)==0) return one/zero; + if((ix|lx)==0) { + if(hx<0) + *signgamp = -1; + return one/vzero; + } if(ix<0x3b900000) { /* |x|<2**-70, return -log(|x|) */ if(hx<0) { *signgamp = -1; @@ -223,9 +227,9 @@ __ieee754_lgamma_r(double x, int *signgamp) } if(hx<0) { if(ix>=0x43300000) /* |x|>=2**52, must be -integer */ - return one/zero; + return one/vzero; t = sin_pi(x); - if(t==zero) return one/zero; /* -integer */ + if(t==zero) return one/vzero; /* -integer */ nadj = __ieee754_log(pi/fabs(t*x)); if(t y) { + z -= 0.25F; /* adjust to round down */ + n--; + } + n &= 7; /* octant of y mod 2 */ + y = y - z + n * 0.25F; /* y mod 2 */ - /* - * argument reduction, make sure inexact flag not raised if input - * is an integer - */ - z = floorf(y); - if(z!=y) { /* inexact anyway */ - y *= (float)0.5; - y = (float)2.0*(y - floorf(y)); /* y = |x| mod 2.0 */ - n = (int) (y*(float)4.0); - } else { - if(ix>=0x4b800000) { - y = zero; n = 0; /* y must be even */ - } else { - if(ix<0x4b000000) z = y+two23; /* exact */ - GET_FLOAT_WORD(n,z); - n &= 1; - y = n; - n<<= 2; - } - } switch (n) { case 0: y = __kernel_sindf(pi*y); break; case 1: @@ -147,7 +132,11 @@ __ieee754_lgammaf_r(float x, int *signgamp) *signgamp = 1; ix = hx&0x7fffffff; if(ix>=0x7f800000) return x*x; - if(ix==0) return one/zero; + if(ix==0) { + if(hx<0) + *signgamp = -1; + return one/vzero; + } if(ix<0x35000000) { /* |x|<2**-21, return -log(|x|) */ if(hx<0) { *signgamp = -1; @@ -156,9 +145,9 @@ __ieee754_lgammaf_r(float x, int *signgamp) } if(hx<0) { if(ix>=0x4b000000) /* |x|>=2**23, must be -integer */ - return one/zero; + return one/vzero; t = sin_pif(x); - if(t==zero) return one/zero; /* -integer */ + if(t==zero) return one/vzero; /* -integer */ nadj = __ieee754_logf(pi/fabsf(t*x)); if(t=0x3fdda618) {y=(float)2.0-x;i=0;} /* [1.7316,2] */ + if(ix>=0x3fdda618) {y=2-x;i=0;} /* [1.7316,2] */ else if(ix>=0x3F9da620) {y=x-tc;i=1;} /* [1.23,1.73] */ else {y=x-one;i=2;} } switch(i) { case 0: z = y*y; - p1 = a0+z*(a2+z*(a4+z*(a6+z*(a8+z*a10)))); - p2 = z*(a1+z*(a3+z*(a5+z*(a7+z*(a9+z*a11))))); + p1 = a0+z*(a2+z*a4); + p2 = z*(a1+z*(a3+z*a5)); p = y*p1+p2; - r += (p-(float)0.5*y); break; + r += (p-y/2); break; case 1: - z = y*y; - w = z*y; - p1 = t0+w*(t3+w*(t6+w*(t9 +w*t12))); /* parallel comp */ - p2 = t1+w*(t4+w*(t7+w*(t10+w*t13))); - p3 = t2+w*(t5+w*(t8+w*(t11+w*t14))); - p = z*p1-(tt-w*(p2+y*p3)); + p = t0+y*t1+y*y*(t2+y*(t3+y*(t4+y*(t5+y*(t6+y*t7))))); r += (tf + p); break; case 2: - p1 = y*(u0+y*(u1+y*(u2+y*(u3+y*(u4+y*u5))))); - p2 = one+y*(v1+y*(v2+y*(v3+y*(v4+y*v5)))); - r += (-(float)0.5*y + p1/p2); + p1 = y*(u0+y*(u1+y*u2)); + p2 = one+y*(v1+y*(v2+y*v3)); + r += (p1/p2-y/2); } } else if(ix<0x41000000) { /* x < 8.0 */ - i = (int)x; - y = x-(float)i; - p = y*(s0+y*(s1+y*(s2+y*(s3+y*(s4+y*(s5+y*s6)))))); - q = one+y*(r1+y*(r2+y*(r3+y*(r4+y*(r5+y*r6))))); - r = half*y+p/q; + i = x; + y = x-i; + p = y*(s0+y*(s1+y*(s2+y*s3))); + q = one+y*(r1+y*(r2+y*r3)); + r = y/2+p/q; z = one; /* lgamma(1+s) = log(s) + lgamma(s) */ switch(i) { - case 7: z *= (y+(float)6.0); /* FALLTHRU */ - case 6: z *= (y+(float)5.0); /* FALLTHRU */ - case 5: z *= (y+(float)4.0); /* FALLTHRU */ - case 4: z *= (y+(float)3.0); /* FALLTHRU */ - case 3: z *= (y+(float)2.0); /* FALLTHRU */ + case 7: z *= (y+6); /* FALLTHRU */ + case 6: z *= (y+5); /* FALLTHRU */ + case 5: z *= (y+4); /* FALLTHRU */ + case 4: z *= (y+3); /* FALLTHRU */ + case 3: z *= (y+2); /* FALLTHRU */ r += __ieee754_logf(z); break; } - /* 8.0 <= x < 2**58 */ - } else if (ix < 0x5c800000) { + /* 8.0 <= x < 2**24 */ + } else if (ix < 0x4b800000) { t = __ieee754_logf(x); z = one/x; y = z*z; - w = w0+z*(w1+y*(w2+y*(w3+y*(w4+y*(w5+y*w6))))); + w = w0+z*(w1+y*w2); r = (x-half)*(t-one)+w; } else - /* 2**58 <= x <= inf */ + /* 2**24 <= x <= inf */ r = x*(__ieee754_logf(x)-one); if(hx<0) r = nadj - r; return r; diff --git a/libm/upstream-freebsd/lib/msun/src/e_lgammal.c b/libm/upstream-freebsd/lib/msun/src/e_lgammal.c new file mode 100644 index 000000000..ebc2fc78c --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/src/e_lgammal.c @@ -0,0 +1,25 @@ +/* @(#)e_lgamma.c 1.3 95/01/18 */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "math.h" +#include "math_private.h" + +extern int signgam; + +long double +lgammal(long double x) +{ + return lgammal_r(x,&signgam); +} diff --git a/libm/upstream-freebsd/lib/msun/src/e_pow.c b/libm/upstream-freebsd/lib/msun/src/e_pow.c index 7607a4a72..d54af9d68 100644 --- a/libm/upstream-freebsd/lib/msun/src/e_pow.c +++ b/libm/upstream-freebsd/lib/msun/src/e_pow.c @@ -19,20 +19,20 @@ __FBSDID("$FreeBSD$"); * 1. Compute and return log2(x) in two pieces: * log2(x) = w1 + w2, * where w1 has 53-24 = 29 bit trailing zeros. - * 2. Perform y*log2(x) = n+y' by simulating muti-precision + * 2. Perform y*log2(x) = n+y' by simulating multi-precision * arithmetic, where |y'|<=0.5. * 3. Return x**y = 2**n*exp(y'*log2) * * Special cases: * 1. (anything) ** 0 is 1 * 2. (anything) ** 1 is itself - * 3. (anything) ** NAN is NAN + * 3. (anything) ** NAN is NAN except 1 ** NAN = 1 * 4. NAN ** (anything except 0) is NAN * 5. +-(|x| > 1) ** +INF is +INF * 6. +-(|x| > 1) ** -INF is +0 * 7. +-(|x| < 1) ** +INF is +0 * 8. +-(|x| < 1) ** -INF is +INF - * 9. +-1 ** +-INF is NAN + * 9. +-1 ** +-INF is 1 * 10. +0 ** (+anything except 0, NAN) is +0 * 11. -0 ** (+anything except 0, NAN, odd integer) is +0 * 12. +0 ** (-anything except 0, NAN) is +INF @@ -141,7 +141,7 @@ __ieee754_pow(double x, double y) if(ly==0) { if (iy==0x7ff00000) { /* y is +-inf */ if(((ix-0x3ff00000)|lx)==0) - return one; /* (-1)**+-inf is NaN */ + return one; /* (-1)**+-inf is 1 */ else if (ix >= 0x3ff00000)/* (|x|>1)**+-inf = inf,0 */ return (hy>=0)? y: zero; else /* (|x|<1)**-,+inf = inf,0 */ diff --git a/libm/upstream-freebsd/lib/msun/src/e_sinh.c b/libm/upstream-freebsd/lib/msun/src/e_sinh.c index 17442d0d1..6c01f4a3c 100644 --- a/libm/upstream-freebsd/lib/msun/src/e_sinh.c +++ b/libm/upstream-freebsd/lib/msun/src/e_sinh.c @@ -32,6 +32,8 @@ __FBSDID("$FreeBSD$"); * only sinh(0)=0 is exact for finite x. */ +#include + #include "math.h" #include "math_private.h" @@ -71,3 +73,7 @@ __ieee754_sinh(double x) /* |x| > overflowthresold, sinh(x) overflow */ return x*shuge; } + +#if (LDBL_MANT_DIG == 53) +__weak_reference(sinh, sinhl); +#endif diff --git a/libm/upstream-freebsd/lib/msun/src/e_sinhl.c b/libm/upstream-freebsd/lib/msun/src/e_sinhl.c new file mode 100644 index 000000000..ce7e3333d --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/src/e_sinhl.c @@ -0,0 +1,131 @@ +/* from: FreeBSD: head/lib/msun/src/e_sinhl.c XXX */ + +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +#include +__FBSDID("$FreeBSD$"); + +/* + * See e_sinh.c for complete comments. + * + * Converted to long double by Bruce D. Evans. + */ + +#include +#ifdef __i386__ +#include +#endif + +#include "fpmath.h" +#include "math.h" +#include "math_private.h" +#include "k_expl.h" + +#if LDBL_MAX_EXP != 0x4000 +/* We also require the usual expsign encoding. */ +#error "Unsupported long double format" +#endif + +#define BIAS (LDBL_MAX_EXP - 1) + +static const long double shuge = 0x1p16383L; +#if LDBL_MANT_DIG == 64 +/* + * Domain [-1, 1], range ~[-6.6749e-22, 6.6749e-22]: + * |sinh(x)/x - s(x)| < 2**-70.3 + */ +static const union IEEEl2bits +S3u = LD80C(0xaaaaaaaaaaaaaaaa, -3, 1.66666666666666666658e-1L); +#define S3 S3u.e +static const double +S5 = 8.3333333333333332e-3, /* 0x11111111111111.0p-59 */ +S7 = 1.9841269841270074e-4, /* 0x1a01a01a01a070.0p-65 */ +S9 = 2.7557319223873889e-6, /* 0x171de3a5565fe6.0p-71 */ +S11 = 2.5052108406704084e-8, /* 0x1ae6456857530f.0p-78 */ +S13 = 1.6059042748655297e-10, /* 0x161245fa910697.0p-85 */ +S15 = 7.6470006914396920e-13, /* 0x1ae7ce4eff2792.0p-93 */ +S17 = 2.8346142308424267e-15; /* 0x19882ce789ffc6.0p-101 */ +#elif LDBL_MANT_DIG == 113 +/* + * Domain [-1, 1], range ~[-2.9673e-36, 2.9673e-36]: + * |sinh(x)/x - s(x)| < 2**-118.0 + */ +static const long double +S3 = 1.66666666666666666666666666666666033e-1L, /* 0x1555555555555555555555555553b.0p-115L */ +S5 = 8.33333333333333333333333333337643193e-3L, /* 0x111111111111111111111111180f5.0p-119L */ +S7 = 1.98412698412698412698412697391263199e-4L, /* 0x1a01a01a01a01a01a01a0176aad11.0p-125L */ +S9 = 2.75573192239858906525574406205464218e-6L, /* 0x171de3a556c7338faac243aaa9592.0p-131L */ +S11 = 2.50521083854417187749675637460977997e-8L, /* 0x1ae64567f544e38fe59b3380d7413.0p-138L */ +S13 = 1.60590438368216146368737762431552702e-10L, /* 0x16124613a86d098059c7620850fc2.0p-145L */ +S15 = 7.64716373181980539786802470969096440e-13L, /* 0x1ae7f3e733b814193af09ce723043.0p-153L */ +S17 = 2.81145725434775409870584280722701574e-15L; /* 0x1952c77030c36898c3fd0b6dfc562.0p-161L */ +static const double +S19= 8.2206352435411005e-18, /* 0x12f49b4662b86d.0p-109 */ +S21= 1.9572943931418891e-20, /* 0x171b8f2fab9628.0p-118 */ +S23 = 3.8679983530666939e-23, /* 0x17617002b73afc.0p-127 */ +S25 = 6.5067867911512749e-26; /* 0x1423352626048a.0p-136 */ +#else +#error "Unsupported long double format" +#endif /* LDBL_MANT_DIG == 64 */ + +/* log(2**16385 - 0.5) rounded up: */ +static const float +o_threshold = 1.13572168e4; /* 0xb174de.0p-10 */ + +long double +sinhl(long double x) +{ + long double hi,lo,x2,x4; + double dx2,s; + int16_t ix,jx; + + GET_LDBL_EXPSIGN(jx,x); + ix = jx&0x7fff; + + /* x is INF or NaN */ + if(ix>=0x7fff) return x+x; + + ENTERI(); + + s = 1; + if (jx<0) s = -1; + + /* |x| < 64, return x, s(x), or accurate s*(exp(|x|)/2-1/exp(|x|)/2) */ + if (ix<0x4005) { /* |x|<64 */ + if (ix1) RETURNI(x); /* sinh(tiny) = tiny with inexact */ + if (ix<0x3fff) { /* |x|<1 */ + x2 = x*x; +#if LDBL_MANT_DIG == 64 + x4 = x2*x2; + RETURNI(((S17*x2 + S15)*x4 + (S13*x2 + S11))*(x2*x*x4*x4) + + ((S9*x2 + S7)*x2 + S5)*(x2*x*x2) + S3*(x2*x) + x); +#elif LDBL_MANT_DIG == 113 + dx2 = x2; + RETURNI(((((((((((S25*dx2 + S23)*dx2 + + S21)*x2 + S19)*x2 + + S17)*x2 + S15)*x2 + S13)*x2 + S11)*x2 + S9)*x2 + S7)*x2 + + S5)* (x2*x*x2) + + S3*(x2*x) + x); +#endif + } + k_hexpl(fabsl(x), &hi, &lo); + RETURNI(s*(lo - 0.25/(hi + lo) + hi)); + } + + /* |x| in [64, o_threshold], return correctly-overflowing s*exp(|x|)/2 */ + if (fabsl(x) <= o_threshold) + RETURNI(s*hexpl(fabsl(x))); + + /* |x| > o_threshold, sinh(x) overflow */ + return x*shuge; +} diff --git a/libm/upstream-freebsd/lib/msun/src/imprecise.c b/libm/upstream-freebsd/lib/msun/src/imprecise.c index a7503bf1c..08cd23980 100644 --- a/libm/upstream-freebsd/lib/msun/src/imprecise.c +++ b/libm/upstream-freebsd/lib/msun/src/imprecise.c @@ -60,10 +60,4 @@ DECLARE_WEAK(powl); long double imprecise_ ## f ## l(long double v) { return f(v); }\ DECLARE_WEAK(f ## l) -DECLARE_IMPRECISE(cosh); -DECLARE_IMPRECISE(erfc); -DECLARE_IMPRECISE(erf); -DECLARE_IMPRECISE(lgamma); -DECLARE_IMPRECISE(sinh); -DECLARE_IMPRECISE(tanh); DECLARE_IMPRECISE(tgamma); diff --git a/libm/upstream-freebsd/lib/msun/src/k_cos.c b/libm/upstream-freebsd/lib/msun/src/k_cos.c index c4702e65b..a0baaacd7 100644 --- a/libm/upstream-freebsd/lib/msun/src/k_cos.c +++ b/libm/upstream-freebsd/lib/msun/src/k_cos.c @@ -68,6 +68,17 @@ C6 = -1.13596475577881948265e-11; /* 0xBDA8FAE9, 0xBE8838D4 */ double __kernel_cos(double x, double y) { +#if defined(QCOM_NEON_OPTIMIZATION) + double hz,z,zz,r,w,k; + + z = x*x; + zz = z*z; + k = x*y; + hz = (float)0.5*z; + r = z*(z*(C1+z*(C2+z*((C3+z*C4)+zz*(C5+z*C6))))); + w = one-hz; + return w + (((one-w)-hz) + (r-k)); +#else double hz,z,r,w; z = x*x; @@ -76,4 +87,5 @@ __kernel_cos(double x, double y) hz = 0.5*z; w = one-hz; return w + (((one-w)-hz) + (z*r-x*y)); +#endif } diff --git a/libm/upstream-freebsd/lib/msun/src/k_sin.c b/libm/upstream-freebsd/lib/msun/src/k_sin.c index 12ee8c143..25b2c5c08 100644 --- a/libm/upstream-freebsd/lib/msun/src/k_sin.c +++ b/libm/upstream-freebsd/lib/msun/src/k_sin.c @@ -59,6 +59,16 @@ S6 = 1.58969099521155010221e-10; /* 0x3DE5D93A, 0x5ACFD57C */ double __kernel_sin(double x, double y, int iy) { +#if defined(QCOM_NEON_OPTIMIZATION) + double z,zz,r,v; + + z = x*x; + zz = z*z; + v = z*x; + r = S2+z*((S3+z*S4)+zz*(S5+z*S6)); + if(iy==0) return x+v*(S1+z*r); + else return x-((z*(half*y-v*r)-y)-v*S1); +#else double z,r,v,w; z = x*x; @@ -67,4 +77,5 @@ __kernel_sin(double x, double y, int iy) v = z*x; if(iy==0) return x+v*(S1+z*r); else return x-((z*(half*y-v*r)-y)-v*S1); +#endif } diff --git a/libm/upstream-freebsd/lib/msun/src/math_private.h b/libm/upstream-freebsd/lib/msun/src/math_private.h index 8af2c650e..a8988d0c7 100644 --- a/libm/upstream-freebsd/lib/msun/src/math_private.h +++ b/libm/upstream-freebsd/lib/msun/src/math_private.h @@ -723,6 +723,16 @@ irintl(long double x) #define __ieee754_remainderf remainderf #define __ieee754_scalbf scalbf +#if defined(QCOM_NEON_OPTIMIZATION) && defined(__ARM_NEON__) +int __kernel_rem_pio2(double*,double*,int,int,int) __attribute__((pcs("aapcs-vfp"))); +//double __full_ieee754_pow(double,double); +#ifndef INLINE_REM_PIO2 +int __ieee754_rem_pio2(double,double*) __attribute__((pcs("aapcs-vfp"))); +#endif +double __kernel_sin(double,double,int) __attribute__((pcs("aapcs-vfp"))); +double __kernel_cos(double,double) __attribute__((pcs("aapcs-vfp"))); +double __kernel_tan(double,double,int) __attribute__((pcs("aapcs-vfp"))); +#else /* fdlibm kernel function */ int __kernel_rem_pio2(double*,double*,int,int,int); @@ -733,6 +743,8 @@ int __ieee754_rem_pio2(double,double*); double __kernel_sin(double,double,int); double __kernel_cos(double,double); double __kernel_tan(double,double,int); +#endif + double __ldexp_exp(double,int); #ifdef _COMPLEX_H double complex __ldexp_cexp(double complex,int); diff --git a/libm/upstream-freebsd/lib/msun/src/s_erf.c b/libm/upstream-freebsd/lib/msun/src/s_erf.c index 0886e5e7d..e1d63bca8 100644 --- a/libm/upstream-freebsd/lib/msun/src/s_erf.c +++ b/libm/upstream-freebsd/lib/msun/src/s_erf.c @@ -111,18 +111,25 @@ __FBSDID("$FreeBSD$"); #include "math.h" #include "math_private.h" +/* XXX Prevent compilers from erroneously constant folding: */ +static const volatile double tiny= 1e-300; + static const double -tiny = 1e-300, -half= 5.00000000000000000000e-01, /* 0x3FE00000, 0x00000000 */ -one = 1.00000000000000000000e+00, /* 0x3FF00000, 0x00000000 */ -two = 2.00000000000000000000e+00, /* 0x40000000, 0x00000000 */ - /* c = (float)0.84506291151 */ +half= 0.5, +one = 1, +two = 2, +/* c = (float)0.84506291151 */ erx = 8.45062911510467529297e-01, /* 0x3FEB0AC1, 0x60000000 */ /* - * Coefficients for approximation to erf on [0,0.84375] + * In the domain [0, 2**-28], only the first term in the power series + * expansion of erf(x) is used. The magnitude of the first neglected + * terms is less than 2**-84. */ efx = 1.28379167095512586316e-01, /* 0x3FC06EBA, 0x8214DB69 */ efx8= 1.02703333676410069053e+00, /* 0x3FF06EBA, 0x8214DB69 */ +/* + * Coefficients for approximation to erf on [0,0.84375] + */ pp0 = 1.28379167095512558561e-01, /* 0x3FC06EBA, 0x8214DB68 */ pp1 = -3.25042107247001499370e-01, /* 0xBFD4CD7D, 0x691CB913 */ pp2 = -2.84817495755985104766e-02, /* 0xBF9D2A51, 0xDBD7194F */ @@ -134,7 +141,7 @@ qq3 = 5.08130628187576562776e-03, /* 0x3F74D022, 0xC4D36B0F */ qq4 = 1.32494738004321644526e-04, /* 0x3F215DC9, 0x221C1A10 */ qq5 = -3.96022827877536812320e-06, /* 0xBED09C43, 0x42A26120 */ /* - * Coefficients for approximation to erf in [0.84375,1.25] + * Coefficients for approximation to erf in [0.84375,1.25] */ pa0 = -2.36211856075265944077e-03, /* 0xBF6359B8, 0xBEF77538 */ pa1 = 4.14856118683748331666e-01, /* 0x3FDA8D00, 0xAD92B34D */ @@ -150,7 +157,7 @@ qa4 = 1.26171219808761642112e-01, /* 0x3FC02660, 0xE763351F */ qa5 = 1.36370839120290507362e-02, /* 0x3F8BEDC2, 0x6B51DD1C */ qa6 = 1.19844998467991074170e-02, /* 0x3F888B54, 0x5735151D */ /* - * Coefficients for approximation to erfc in [1.25,1/0.35] + * Coefficients for approximation to erfc in [1.25,1/0.35] */ ra0 = -9.86494403484714822705e-03, /* 0xBF843412, 0x600D6435 */ ra1 = -6.93858572707181764372e-01, /* 0xBFE63416, 0xE4BA7360 */ @@ -169,7 +176,7 @@ sa6 = 1.08635005541779435134e+02, /* 0x405B28A3, 0xEE48AE2C */ sa7 = 6.57024977031928170135e+00, /* 0x401A47EF, 0x8E484A93 */ sa8 = -6.04244152148580987438e-02, /* 0xBFAEEFF2, 0xEE749A62 */ /* - * Coefficients for approximation to erfc in [1/.35,28] + * Coefficients for approximation to erfc in [1/.35,28] */ rb0 = -9.86494292470009928597e-03, /* 0xBF843412, 0x39E86F4A */ rb1 = -7.99283237680523006574e-01, /* 0xBFE993BA, 0x70C285DE */ @@ -201,7 +208,7 @@ erf(double x) if(ix < 0x3feb0000) { /* |x|<0.84375 */ if(ix < 0x3e300000) { /* |x|<2**-28 */ if (ix < 0x00800000) - return 0.125*(8.0*x+efx8*x); /*avoid underflow */ + return (8*x+efx8*x)/8; /* avoid spurious underflow */ return x + efx*x; } z = x*x; @@ -222,15 +229,12 @@ erf(double x) x = fabs(x); s = one/(x*x); if(ix< 0x4006DB6E) { /* |x| < 1/0.35 */ - R=ra0+s*(ra1+s*(ra2+s*(ra3+s*(ra4+s*( - ra5+s*(ra6+s*ra7)))))); - S=one+s*(sa1+s*(sa2+s*(sa3+s*(sa4+s*( - sa5+s*(sa6+s*(sa7+s*sa8))))))); + R=ra0+s*(ra1+s*(ra2+s*(ra3+s*(ra4+s*(ra5+s*(ra6+s*ra7)))))); + S=one+s*(sa1+s*(sa2+s*(sa3+s*(sa4+s*(sa5+s*(sa6+s*(sa7+ + s*sa8))))))); } else { /* |x| >= 1/0.35 */ - R=rb0+s*(rb1+s*(rb2+s*(rb3+s*(rb4+s*( - rb5+s*rb6))))); - S=one+s*(sb1+s*(sb2+s*(sb3+s*(sb4+s*( - sb5+s*(sb6+s*sb7)))))); + R=rb0+s*(rb1+s*(rb2+s*(rb3+s*(rb4+s*(rb5+s*rb6))))); + S=one+s*(sb1+s*(sb2+s*(sb3+s*(sb4+s*(sb5+s*(sb6+s*sb7)))))); } z = x; SET_LOW_WORD(z,0); @@ -238,6 +242,10 @@ erf(double x) if(hx>=0) return one-r/x; else return r/x-one; } +#if (LDBL_MANT_DIG == 53) +__weak_reference(erf, erfl); +#endif + double erfc(double x) { @@ -279,23 +287,23 @@ erfc(double x) x = fabs(x); s = one/(x*x); if(ix< 0x4006DB6D) { /* |x| < 1/.35 ~ 2.857143*/ - R=ra0+s*(ra1+s*(ra2+s*(ra3+s*(ra4+s*( - ra5+s*(ra6+s*ra7)))))); - S=one+s*(sa1+s*(sa2+s*(sa3+s*(sa4+s*( - sa5+s*(sa6+s*(sa7+s*sa8))))))); + R=ra0+s*(ra1+s*(ra2+s*(ra3+s*(ra4+s*(ra5+s*(ra6+s*ra7)))))); + S=one+s*(sa1+s*(sa2+s*(sa3+s*(sa4+s*(sa5+s*(sa6+s*(sa7+ + s*sa8))))))); } else { /* |x| >= 1/.35 ~ 2.857143 */ if(hx<0&&ix>=0x40180000) return two-tiny;/* x < -6 */ - R=rb0+s*(rb1+s*(rb2+s*(rb3+s*(rb4+s*( - rb5+s*rb6))))); - S=one+s*(sb1+s*(sb2+s*(sb3+s*(sb4+s*( - sb5+s*(sb6+s*sb7)))))); + R=rb0+s*(rb1+s*(rb2+s*(rb3+s*(rb4+s*(rb5+s*rb6))))); + S=one+s*(sb1+s*(sb2+s*(sb3+s*(sb4+s*(sb5+s*(sb6+s*sb7)))))); } z = x; SET_LOW_WORD(z,0); - r = __ieee754_exp(-z*z-0.5625)* - __ieee754_exp((z-x)*(z+x)+R/S); + r = __ieee754_exp(-z*z-0.5625)*__ieee754_exp((z-x)*(z+x)+R/S); if(hx>0) return r/x; else return two-r/x; } else { if(hx>0) return tiny*tiny; else return two-tiny; } } + +#if (LDBL_MANT_DIG == 53) +__weak_reference(erfc, erfcl); +#endif diff --git a/libm/upstream-freebsd/lib/msun/src/s_erff.c b/libm/upstream-freebsd/lib/msun/src/s_erff.c index a3579f1a0..d6cfbd227 100644 --- a/libm/upstream-freebsd/lib/msun/src/s_erff.c +++ b/libm/upstream-freebsd/lib/msun/src/s_erff.c @@ -19,64 +19,63 @@ __FBSDID("$FreeBSD$"); #include "math.h" #include "math_private.h" +/* XXX Prevent compilers from erroneously constant folding: */ +static const volatile float tiny = 1e-30; + static const float -tiny = 1e-30, -half= 5.0000000000e-01, /* 0x3F000000 */ -one = 1.0000000000e+00, /* 0x3F800000 */ -two = 2.0000000000e+00, /* 0x40000000 */ +half= 0.5, +one = 1, +two = 2, +erx = 8.42697144e-01, /* 0x3f57bb00 */ /* - * Coefficients for approximation to erf on [0,0.84375] + * In the domain [0, 2**-14], only the first term in the power series + * expansion of erf(x) is used. The magnitude of the first neglected + * terms is less than 2**-42. */ -efx = 1.2837916613e-01, /* 0x3e0375d4 */ -efx8= 1.0270333290e+00, /* 0x3f8375d4 */ +efx = 1.28379166e-01, /* 0x3e0375d4 */ +efx8= 1.02703333e+00, /* 0x3f8375d4 */ /* - * Domain [0, 0.84375], range ~[-5.4446e-10,5.5197e-10]: - * |(erf(x) - x)/x - p(x)/q(x)| < 2**-31. + * Domain [0, 0.84375], range ~[-5.4419e-10, 5.5179e-10]: + * |(erf(x) - x)/x - pp(x)/qq(x)| < 2**-31 */ -pp0 = 1.28379166e-01F, /* 0x1.06eba8p-3 */ -pp1 = -3.36030394e-01F, /* -0x1.58185ap-2 */ -pp2 = -1.86260219e-03F, /* -0x1.e8451ep-10 */ -qq1 = 3.12324286e-01F, /* 0x1.3fd1f0p-2 */ -qq2 = 2.16070302e-02F, /* 0x1.620274p-6 */ -qq3 = -1.98859419e-03F, /* -0x1.04a626p-9 */ +pp0 = 1.28379166e-01, /* 0x3e0375d4 */ +pp1 = -3.36030394e-01, /* 0xbeac0c2d */ +pp2 = -1.86261395e-03, /* 0xbaf422f4 */ +qq1 = 3.12324315e-01, /* 0x3e9fe8f9 */ +qq2 = 2.16070414e-02, /* 0x3cb10140 */ +qq3 = -1.98859372e-03, /* 0xbb025311 */ /* - * Domain [0.84375, 1.25], range ~[-1.953e-11,1.940e-11]: - * |(erf(x) - erx) - p(x)/q(x)| < 2**-36. + * Domain [0.84375, 1.25], range ~[-1.023e-9, 1.023e-9]: + * |(erf(x) - erx) - pa(x)/qa(x)| < 2**-31 */ -erx = 8.42697144e-01F, /* 0x1.af7600p-1. erf(1) rounded to 16 bits. */ -pa0 = 3.64939137e-06F, /* 0x1.e9d022p-19 */ -pa1 = 4.15109694e-01F, /* 0x1.a91284p-2 */ -pa2 = -1.65179938e-01F, /* -0x1.5249dcp-3 */ -pa3 = 1.10914491e-01F, /* 0x1.c64e46p-4 */ -qa1 = 6.02074385e-01F, /* 0x1.344318p-1 */ -qa2 = 5.35934687e-01F, /* 0x1.126608p-1 */ -qa3 = 1.68576106e-01F, /* 0x1.593e6ep-3 */ -qa4 = 5.62181212e-02F, /* 0x1.cc89f2p-5 */ +pa0 = 3.65041046e-06, /* 0x3674f993 */ +pa1 = 4.15109307e-01, /* 0x3ed48935 */ +pa2 = -2.09395722e-01, /* 0xbe566bd5 */ +pa3 = 8.67677554e-02, /* 0x3db1b34b */ +qa1 = 4.95560974e-01, /* 0x3efdba2b */ +qa2 = 3.71248513e-01, /* 0x3ebe1449 */ +qa3 = 3.92478965e-02, /* 0x3d20c267 */ /* - * Domain [1.25,1/0.35], range ~[-7.043e-10,7.457e-10]: - * |log(x*erfc(x)) + x**2 + 0.5625 - r(x)/s(x)| < 2**-30 + * Domain [1.25,1/0.35], range ~[-4.821e-9, 4.927e-9]: + * |log(x*erfc(x)) + x**2 + 0.5625 - ra(x)/sa(x)| < 2**-28 */ -ra0 = -9.87132732e-03F, /* -0x1.4376b2p-7 */ -ra1 = -5.53605914e-01F, /* -0x1.1b723cp-1 */ -ra2 = -2.17589188e+00F, /* -0x1.1683a0p+1 */ -ra3 = -1.43268085e+00F, /* -0x1.6ec42cp+0 */ -sa1 = 5.45995426e+00F, /* 0x1.5d6fe4p+2 */ -sa2 = 6.69798088e+00F, /* 0x1.acabb8p+2 */ -sa3 = 1.43113089e+00F, /* 0x1.6e5e98p+0 */ -sa4 = -5.77397496e-02F, /* -0x1.d90108p-5 */ +ra0 = -9.88156721e-03, /* 0xbc21e64c */ +ra1 = -5.43658376e-01, /* 0xbf0b2d32 */ +ra2 = -1.66828310e+00, /* 0xbfd58a4d */ +ra3 = -6.91554189e-01, /* 0xbf3109b2 */ +sa1 = 4.48581553e+00, /* 0x408f8bcd */ +sa2 = 4.10799170e+00, /* 0x408374ab */ +sa3 = 5.53855181e-01, /* 0x3f0dc974 */ /* - * Domain [1/0.35, 11], range ~[-2.264e-13,2.336e-13]: - * |log(x*erfc(x)) + x**2 + 0.5625 - r(x)/s(x)| < 2**-42 + * Domain [2.85715, 11], range ~[-1.484e-9, 1.505e-9]: + * |log(x*erfc(x)) + x**2 + 0.5625 - rb(x)/sb(x)| < 2**-30 */ -rb0 = -9.86494310e-03F, /* -0x1.434124p-7 */ -rb1 = -6.25171244e-01F, /* -0x1.401672p-1 */ -rb2 = -6.16498327e+00F, /* -0x1.8a8f16p+2 */ -rb3 = -1.66696873e+01F, /* -0x1.0ab70ap+4 */ -rb4 = -9.53764343e+00F, /* -0x1.313460p+3 */ -sb1 = 1.26884899e+01F, /* 0x1.96081cp+3 */ -sb2 = 4.51839523e+01F, /* 0x1.6978bcp+5 */ -sb3 = 4.72810211e+01F, /* 0x1.7a3f88p+5 */ -sb4 = 8.93033314e+00F; /* 0x1.1dc54ap+3 */ +rb0 = -9.86496918e-03, /* 0xbc21a0ae */ +rb1 = -5.48049808e-01, /* 0xbf0c4cfe */ +rb2 = -1.84115684e+00, /* 0xbfebab07 */ +sb1 = 4.87132740e+00, /* 0x409be1ea */ +sb2 = 3.04982710e+00, /* 0x4043305e */ +sb3 = -7.61900663e-01; /* 0xbf430bec */ float erff(float x) @@ -85,9 +84,9 @@ erff(float x) float R,S,P,Q,s,y,z,r; GET_FLOAT_WORD(hx,x); ix = hx&0x7fffffff; - if(ix>=0x7f800000) { /* erf(nan)=nan */ + if(ix>=0x7f800000) { /* erff(nan)=nan */ i = ((u_int32_t)hx>>31)<<1; - return (float)(1-i)+one/x; /* erf(+-inf)=+-1 */ + return (float)(1-i)+one/x; /* erff(+-inf)=+-1 */ } if(ix < 0x3f580000) { /* |x|<0.84375 */ @@ -105,7 +104,7 @@ erff(float x) if(ix < 0x3fa00000) { /* 0.84375 <= |x| < 1.25 */ s = fabsf(x)-one; P = pa0+s*(pa1+s*(pa2+s*pa3)); - Q = one+s*(qa1+s*(qa2+s*(qa3+s*qa4))); + Q = one+s*(qa1+s*(qa2+s*qa3)); if(hx>=0) return erx + P/Q; else return -erx - P/Q; } if (ix >= 0x40800000) { /* inf>|x|>=4 */ @@ -113,12 +112,12 @@ erff(float x) } x = fabsf(x); s = one/(x*x); - if(ix< 0x4036DB6E) { /* |x| < 1/0.35 */ + if(ix< 0x4036db8c) { /* |x| < 2.85715 ~ 1/0.35 */ R=ra0+s*(ra1+s*(ra2+s*ra3)); - S=one+s*(sa1+s*(sa2+s*(sa3+s*sa4))); - } else { /* |x| >= 1/0.35 */ - R=rb0+s*(rb1+s*(rb2+s*(rb3+s*rb4))); - S=one+s*(sb1+s*(sb2+s*(sb3+s*sb4))); + S=one+s*(sa1+s*(sa2+s*sa3)); + } else { /* |x| >= 2.85715 ~ 1/0.35 */ + R=rb0+s*(rb1+s*rb2); + S=one+s*(sb1+s*(sb2+s*sb3)); } SET_FLOAT_WORD(z,hx&0xffffe000); r = expf(-z*z-0.5625F)*expf((z-x)*(z+x)+R/S); @@ -132,8 +131,8 @@ erfcf(float x) float R,S,P,Q,s,y,z,r; GET_FLOAT_WORD(hx,x); ix = hx&0x7fffffff; - if(ix>=0x7f800000) { /* erfc(nan)=nan */ - /* erfc(+-inf)=0,2 */ + if(ix>=0x7f800000) { /* erfcf(nan)=nan */ + /* erfcf(+-inf)=0,2 */ return (float)(((u_int32_t)hx>>31)<<1)+one/x; } @@ -155,7 +154,7 @@ erfcf(float x) if(ix < 0x3fa00000) { /* 0.84375 <= |x| < 1.25 */ s = fabsf(x)-one; P = pa0+s*(pa1+s*(pa2+s*pa3)); - Q = one+s*(qa1+s*(qa2+s*(qa3+s*qa4))); + Q = one+s*(qa1+s*(qa2+s*qa3)); if(hx>=0) { z = one-erx; return z - P/Q; } else { @@ -165,13 +164,13 @@ erfcf(float x) if (ix < 0x41300000) { /* |x|<11 */ x = fabsf(x); s = one/(x*x); - if(ix< 0x4036DB6D) { /* |x| < 1/.35 ~ 2.857143*/ - R=ra0+s*(ra1+s*(ra2+s*ra3)); - S=one+s*(sa1+s*(sa2+s*(sa3+s*sa4))); - } else { /* |x| >= 1/.35 ~ 2.857143 */ + if(ix< 0x4036db8c) { /* |x| < 2.85715 ~ 1/.35 */ + R=ra0+s*(ra1+s*(ra2+s*ra3)); + S=one+s*(sa1+s*(sa2+s*sa3)); + } else { /* |x| >= 2.85715 ~ 1/.35 */ if(hx<0&&ix>=0x40a00000) return two-tiny;/* x < -5 */ - R=rb0+s*(rb1+s*(rb2+s*(rb3+s*rb4))); - S=one+s*(sb1+s*(sb2+s*(sb3+s*sb4))); + R=rb0+s*(rb1+s*rb2); + S=one+s*(sb1+s*(sb2+s*sb3)); } SET_FLOAT_WORD(z,hx&0xffffe000); r = expf(-z*z-0.5625F)*expf((z-x)*(z+x)+R/S); diff --git a/libm/upstream-freebsd/lib/msun/src/s_rintl.c b/libm/upstream-freebsd/lib/msun/src/s_rintl.c index e55b40e5b..b43df89f5 100644 --- a/libm/upstream-freebsd/lib/msun/src/s_rintl.c +++ b/libm/upstream-freebsd/lib/msun/src/s_rintl.c @@ -29,7 +29,6 @@ __FBSDID("$FreeBSD$"); #include #include -#include #include "fpmath.h" diff --git a/libm/upstream-freebsd/lib/msun/src/s_tanh.c b/libm/upstream-freebsd/lib/msun/src/s_tanh.c index 96e35656b..6d26c695d 100644 --- a/libm/upstream-freebsd/lib/msun/src/s_tanh.c +++ b/libm/upstream-freebsd/lib/msun/src/s_tanh.c @@ -37,10 +37,13 @@ __FBSDID("$FreeBSD$"); * only tanh(0)=0 is exact for finite argument. */ +#include + #include "math.h" #include "math_private.h" -static const double one = 1.0, two = 2.0, tiny = 1.0e-300, huge = 1.0e300; +static const volatile double tiny = 1.0e-300; +static const double one = 1.0, two = 2.0, huge = 1.0e300; double tanh(double x) @@ -75,3 +78,7 @@ tanh(double x) } return (jx>=0)? z: -z; } + +#if (LDBL_MANT_DIG == 53) +__weak_reference(tanh, tanhl); +#endif diff --git a/libm/upstream-freebsd/lib/msun/src/s_tanhf.c b/libm/upstream-freebsd/lib/msun/src/s_tanhf.c index 04f09c686..f537be4fd 100644 --- a/libm/upstream-freebsd/lib/msun/src/s_tanhf.c +++ b/libm/upstream-freebsd/lib/msun/src/s_tanhf.c @@ -19,7 +19,9 @@ __FBSDID("$FreeBSD$"); #include "math.h" #include "math_private.h" -static const float one=1.0, two=2.0, tiny = 1.0e-30, huge = 1.0e30; +static const volatile float tiny = 1.0e-30; +static const float one=1.0, two=2.0, huge = 1.0e30; + float tanhf(float x) { diff --git a/libm/upstream-freebsd/lib/msun/src/s_tanhl.c b/libm/upstream-freebsd/lib/msun/src/s_tanhl.c new file mode 100644 index 000000000..886158b78 --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/src/s_tanhl.c @@ -0,0 +1,172 @@ +/* from: FreeBSD: head/lib/msun/src/s_tanhl.c XXX */ + +/* @(#)s_tanh.c 5.1 93/09/24 */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +#include +__FBSDID("$FreeBSD$"); + +/* + * See s_tanh.c for complete comments. + * + * Converted to long double by Bruce D. Evans. + */ + +#include +#ifdef __i386__ +#include +#endif + +#include "math.h" +#include "math_private.h" +#include "fpmath.h" +#include "k_expl.h" + +#if LDBL_MAX_EXP != 0x4000 +/* We also require the usual expsign encoding. */ +#error "Unsupported long double format" +#endif + +#define BIAS (LDBL_MAX_EXP - 1) + +static const volatile double tiny = 1.0e-300; +static const double one = 1.0; +#if LDBL_MANT_DIG == 64 +/* + * Domain [-0.25, 0.25], range ~[-1.6304e-22, 1.6304e-22]: + * |tanh(x)/x - t(x)| < 2**-72.3 + */ +static const union IEEEl2bits +T3u = LD80C(0xaaaaaaaaaaaaaa9f, -2, -3.33333333333333333017e-1L); +#define T3 T3u.e +static const double +T5 = 1.3333333333333314e-1, /* 0x1111111111110a.0p-55 */ +T7 = -5.3968253968210485e-2, /* -0x1ba1ba1ba1a1a1.0p-57 */ +T9 = 2.1869488531393817e-2, /* 0x1664f488172022.0p-58 */ +T11 = -8.8632352345964591e-3, /* -0x1226e34bc138d5.0p-59 */ +T13 = 3.5921169709993771e-3, /* 0x1d6d371d3e400f.0p-61 */ +T15 = -1.4555786415756001e-3, /* -0x17d923aa63814d.0p-62 */ +T17 = 5.8645267876296793e-4, /* 0x13378589b85aa7.0p-63 */ +T19 = -2.1121033571392224e-4; /* -0x1baf0af80c4090.0p-65 */ +#elif LDBL_MANT_DIG == 113 +/* + * Domain [-0.25, 0.25], range ~[-2.4211e-37, 2.4211e-37]: + * |tanh(x)/x - t(x)| < 2**121.6 + */ +static const long double +T3 = -3.33333333333333333333333333333332980e-1L, /* -0x1555555555555555555555555554e.0p-114L */ +T5 = 1.33333333333333333333333333332707260e-1L, /* 0x1111111111111111111111110ab7b.0p-115L */ +T7 = -5.39682539682539682539682535723482314e-2L, /* -0x1ba1ba1ba1ba1ba1ba1ba17b5fc98.0p-117L */ +T9 = 2.18694885361552028218693591149061717e-2L, /* 0x1664f4882c10f9f32d6b1a12a25e5.0p-118L */ +T11 = -8.86323552990219656883762347736381851e-3L, /* -0x1226e355e6c23c8f5a5a0f386cb4d.0p-119L */ +T13 = 3.59212803657248101358314398220822722e-3L, /* 0x1d6d3d0e157ddfb403ad3637442c6.0p-121L */ +T15 = -1.45583438705131796512568010348874662e-3L; /* -0x17da36452b75e150c44cc34253b34.0p-122L */ +static const double +T17 = 5.9002744094556621e-4, /* 0x1355824803668e.0p-63 */ +T19 = -2.3912911424260516e-4, /* -0x1f57d7734c8dde.0p-65 */ +T21 = 9.6915379535512898e-5, /* 0x1967e18ad6a6ca.0p-66 */ +T23 = -3.9278322983156353e-5, /* -0x1497d8e6b75729.0p-67 */ +T25 = 1.5918887220143869e-5, /* 0x10b1319998cafa.0p-68 */ +T27 = -6.4514295231630956e-6, /* -0x1b0f2b71b218eb.0p-70 */ +T29 = 2.6120754043964365e-6, /* 0x15e963a3cf3a39.0p-71 */ +T31 = -1.0407567231003314e-6, /* -0x1176041e656869.0p-72 */ +T33 = 3.4744117554063574e-7; /* 0x1750fe732cab9c.0p-74 */ +#endif /* LDBL_MANT_DIG == 64 */ + +static inline long double +divl(long double a, long double b, long double c, long double d, + long double e, long double f) +{ + long double inv, r; + float fr, fw; + + _2sumF(a, c); + b = b + c; + _2sumF(d, f); + e = e + f; + + inv = 1 / (d + e); + + r = (a + b) * inv; + fr = r; + r = fr; + + fw = d + e; + e = d - fw + e; + d = fw; + + r = r + (a - d * r + b - e * r) * inv; + + return r; +} + +long double +tanhl(long double x) +{ + long double hi,lo,s,x2,x4,z; + double dx2; + int16_t jx,ix; + + GET_LDBL_EXPSIGN(jx,x); + ix = jx&0x7fff; + + /* x is INF or NaN */ + if(ix>=0x7fff) { + if (jx>=0) return one/x+one; /* tanh(+-inf)=+-1 */ + else return one/x-one; /* tanh(NaN) = NaN */ + } + + ENTERI(); + + /* |x| < 40 */ + if (ix < 0x4004 || fabsl(x) < 40) { /* |x|<40 */ + if (__predict_false(ix= 40, return +-1 */ + } else { + z = one - tiny; /* raise inexact flag */ + } + s = 1; + if (jx<0) s = -1; + RETURNI(s*z); +} diff --git a/linker/Android.mk b/linker/Android.mk index 4298032a4..52f3139cc 100644 --- a/linker/Android.mk +++ b/linker/Android.mk @@ -26,7 +26,7 @@ LOCAL_LDFLAGS := \ LOCAL_CFLAGS += \ -fno-stack-protector \ - -Wstrict-overflow=5 \ + -Wstrict-overflow=2 \ -fvisibility=hidden \ -Wall -Wextra -Wunused -Werror \ diff --git a/tests/Android.mk b/tests/Android.mk index 8b0b0a0e0..2db088d3f 100644 --- a/tests/Android.mk +++ b/tests/Android.mk @@ -54,6 +54,7 @@ test_cppflags = \ libBionicStandardTests_src_files := \ arpa_inet_test.cpp \ buffer_tests.cpp \ + complex_test.cpp \ ctype_test.cpp \ dirent_test.cpp \ eventfd_test.cpp \ @@ -85,6 +86,7 @@ libBionicStandardTests_src_files := \ mntent_test.cpp \ netdb_test.cpp \ pthread_test.cpp \ + pty_test.cpp \ regex_test.cpp \ sched_test.cpp \ search_test.cpp \ @@ -116,6 +118,7 @@ libBionicStandardTests_src_files := \ uchar_test.cpp \ uniqueptr_test.cpp \ unistd_test.cpp \ + utmp_test.cpp \ wchar_test.cpp \ libBionicStandardTests_cflags := \ @@ -291,7 +294,7 @@ bionic-unit-tests-glibc_whole_static_libraries := \ libBionicStandardTests \ bionic-unit-tests-glibc_ldlibs := \ - -lrt -ldl \ + -lrt -ldl -lutil \ bionic-unit-tests-glibc_c_includes := \ bionic/libc \ diff --git a/tests/complex_test.cpp b/tests/complex_test.cpp new file mode 100644 index 000000000..47964f6b9 --- /dev/null +++ b/tests/complex_test.cpp @@ -0,0 +1,260 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +// libc++ actively gets in the way of including from C++, so we +// have to declare the complex math functions ourselves. +// (libc++ also seems to have really bad implementations of its own that ignore +// the intricacies of floating point math.) +// http://llvm.org/bugs/show_bug.cgi?id=21504 + +#include // For M_PI. + +extern "C" double cabs(double _Complex); +TEST(complex, cabs) { + ASSERT_EQ(0.0, cabs(0)); +} + +extern "C" float cabsf(float _Complex); +TEST(complex, cabsf) { + ASSERT_EQ(0.0, cabsf(0)); +} + +extern "C" long double cabsl(long double _Complex); +TEST(complex, cabsl) { + ASSERT_EQ(0.0, cabsl(0)); +} + +extern "C" double _Complex cacos(double _Complex); +TEST(complex, cacos) { + ASSERT_EQ(M_PI/2.0, cacos(0.0)); +} + +extern "C" float _Complex cacosf(float _Complex); +TEST(complex, cacosf) { + ASSERT_EQ(static_cast(M_PI)/2.0f, cacosf(0.0)); +} + +extern "C" double _Complex cacosh(double _Complex); +TEST(complex, cacosh) { + ASSERT_EQ(0.0, cacosh(1.0)); +} + +extern "C" float _Complex cacoshf(float _Complex); +TEST(complex, cacoshf) { + ASSERT_EQ(0.0, cacoshf(1.0)); +} + +extern "C" double carg(double _Complex); +TEST(complex, carg) { + ASSERT_EQ(0.0, carg(0)); +} + +extern "C" float cargf(float _Complex); +TEST(complex, cargf) { + ASSERT_EQ(0.0, cargf(0)); +} + +extern "C" long double cargl(long double _Complex); +TEST(complex, cargl) { + ASSERT_EQ(0.0, cargl(0)); +} + +extern "C" double _Complex casin(double _Complex); +TEST(complex, casin) { + ASSERT_EQ(0.0, casin(0)); +} + +extern "C" float _Complex casinf(float _Complex); +TEST(complex, casinf) { + ASSERT_EQ(0.0, casinf(0)); +} + +extern "C" double _Complex casinh(double _Complex); +TEST(complex, casinh) { + ASSERT_EQ(0.0, casinh(0)); +} + +extern "C" float _Complex casinhf(float _Complex); +TEST(complex, casinhf) { + ASSERT_EQ(0.0, casinhf(0)); +} + +extern "C" double _Complex catan(double _Complex); +TEST(complex, catan) { + ASSERT_EQ(0.0, catan(0)); +} + +extern "C" float _Complex catanf(float _Complex); +TEST(complex, catanf) { + ASSERT_EQ(0.0, catanf(0)); +} + +extern "C" double _Complex catanh(double _Complex); +TEST(complex, catanh) { + ASSERT_EQ(0.0, catanh(0)); +} + +extern "C" float _Complex catanhf(float _Complex); +TEST(complex, catanhf) { + ASSERT_EQ(0.0, catanhf(0)); +} + +extern "C" double _Complex ccos(double _Complex); +TEST(complex, ccos) { + ASSERT_EQ(1.0, ccos(0)); +} + +extern "C" float _Complex ccosf(float _Complex); +TEST(complex, ccosf) { + ASSERT_EQ(1.0, ccosf(0)); +} + +extern "C" double _Complex ccosh(double _Complex); +TEST(complex, ccosh) { + ASSERT_EQ(1.0, ccosh(0)); +} + +extern "C" float _Complex ccoshf(float _Complex); +TEST(complex, ccoshf) { + ASSERT_EQ(1.0, ccoshf(0)); +} + +extern "C" double _Complex cexp(double _Complex); +TEST(complex, cexp) { + ASSERT_EQ(1.0, cexp(0)); +} + +extern "C" float _Complex cexpf(float _Complex); +TEST(complex, cexpf) { + ASSERT_EQ(1.0, cexpf(0)); +} + +extern "C" double cimag(double _Complex); +TEST(complex, cimag) { + ASSERT_EQ(0.0, cimag(0)); +} + +extern "C" float cimagf(float _Complex); +TEST(complex, cimagf) { + ASSERT_EQ(0.0f, cimagf(0)); +} + +extern "C" long double cimagl(long double _Complex); +TEST(complex, cimagl) { + ASSERT_EQ(0.0, cimagl(0)); +} + +extern "C" double _Complex conj(double _Complex); +TEST(complex, conj) { + ASSERT_EQ(0.0, conj(0)); +} + +extern "C" float _Complex conjf(float _Complex); +TEST(complex, conjf) { + ASSERT_EQ(0.0f, conjf(0)); +} + +extern "C" long double _Complex conjl(long double _Complex); +TEST(complex, conjl) { + ASSERT_EQ(0.0, conjl(0)); +} + +extern "C" double _Complex cproj(double _Complex); +TEST(complex, cproj) { + ASSERT_EQ(0.0, cproj(0)); +} + +extern "C" float _Complex cprojf(float _Complex); +TEST(complex, cprojf) { + ASSERT_EQ(0.0f, cprojf(0)); +} + +extern "C" long double _Complex cprojl(long double _Complex); +TEST(complex, cprojl) { + ASSERT_EQ(0.0, cprojl(0)); +} + +extern "C" double creal(double _Complex); +TEST(complex, creal) { + ASSERT_EQ(0.0, creal(0)); +} + +extern "C" float crealf(float _Complex); +TEST(complex, crealf) { + ASSERT_EQ(0.0f, crealf(0)); +} + +extern "C" long double creall(long double _Complex); +TEST(complex, creall) { + ASSERT_EQ(0.0, creall(0)); +} + +extern "C" double _Complex csin(double _Complex); +TEST(complex, csin) { + ASSERT_EQ(0.0, csin(0)); +} + +extern "C" float _Complex csinf(float _Complex); +TEST(complex, csinf) { + ASSERT_EQ(0.0, csinf(0)); +} + +extern "C" double _Complex csinh(double _Complex); +TEST(complex, csinh) { + ASSERT_EQ(0.0, csinh(0)); +} + +extern "C" float _Complex csinhf(float _Complex); +TEST(complex, csinhf) { + ASSERT_EQ(0.0, csinhf(0)); +} + +extern "C" double _Complex csqrt(double _Complex); +TEST(complex, csqrt) { + ASSERT_EQ(0.0, csqrt(0)); +} + +extern "C" float _Complex csqrtf(float _Complex); +TEST(complex, csqrtf) { + ASSERT_EQ(0.0f, csqrt(0)); +} + +extern "C" long double _Complex csqrtl(long double _Complex); +TEST(complex, csqrtl) { + ASSERT_EQ(0.0, csqrtl(0)); +} + +extern "C" double _Complex ctan(double _Complex); +TEST(complex, ctan) { + ASSERT_EQ(0.0, ctan(0)); +} + +extern "C" float _Complex ctanf(float _Complex); +TEST(complex, ctanf) { + ASSERT_EQ(0.0, ctanf(0)); +} + +extern "C" double _Complex ctanh(double _Complex); +TEST(complex, ctanh) { + ASSERT_EQ(0.0, ctanh(0)); +} + +extern "C" float _Complex ctanhf(float _Complex); +TEST(complex, ctanhf) { + ASSERT_EQ(0.0, ctanhf(0)); +} diff --git a/tests/math_test.cpp b/tests/math_test.cpp index 2203db9f4..d0fbc1f8f 100644 --- a/tests/math_test.cpp +++ b/tests/math_test.cpp @@ -1188,12 +1188,51 @@ TEST(math, lgamma_r) { ASSERT_EQ(1, sign); } +TEST(math, lgamma_r_17471883) { + int sign; + + sign = 0; + ASSERT_DOUBLE_EQ(HUGE_VAL, lgamma_r(0.0, &sign)); + ASSERT_EQ(1, sign); + sign = 0; + ASSERT_DOUBLE_EQ(HUGE_VAL, lgamma_r(-0.0, &sign)); + ASSERT_EQ(-1, sign); +} + TEST(math, lgammaf_r) { int sign; ASSERT_FLOAT_EQ(logf(24.0f), lgammaf_r(5.0f, &sign)); ASSERT_EQ(1, sign); } +TEST(math, lgammaf_r_17471883) { + int sign; + + sign = 0; + ASSERT_FLOAT_EQ(HUGE_VALF, lgammaf_r(0.0f, &sign)); + ASSERT_EQ(1, sign); + sign = 0; + ASSERT_FLOAT_EQ(HUGE_VALF, lgammaf_r(-0.0f, &sign)); + ASSERT_EQ(-1, sign); +} + +TEST(math, lgammal_r) { + int sign; + ASSERT_DOUBLE_EQ(log(24.0L), lgamma_r(5.0L, &sign)); + ASSERT_EQ(1, sign); +} + +TEST(math, lgammal_r_17471883) { + int sign; + + sign = 0; + ASSERT_DOUBLE_EQ(HUGE_VAL, lgammal_r(0.0L, &sign)); + ASSERT_EQ(1, sign); + sign = 0; + ASSERT_DOUBLE_EQ(HUGE_VAL, lgammal_r(-0.0L, &sign)); + ASSERT_EQ(-1, sign); +} + TEST(math, tgamma) { ASSERT_DOUBLE_EQ(24.0, tgamma(5.0)); } diff --git a/tests/pty_test.cpp b/tests/pty_test.cpp new file mode 100644 index 000000000..7fe97e9af --- /dev/null +++ b/tests/pty_test.cpp @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +TEST(pty, openpty) { + int master, slave; + char name[32]; + struct winsize w = { 123, 456, 9999, 999 }; + ASSERT_EQ(0, openpty(&master, &slave, name, NULL, &w)); + ASSERT_NE(-1, master); + ASSERT_NE(-1, slave); + ASSERT_NE(master, slave); + + char tty_name[32]; + ASSERT_EQ(0, ttyname_r(slave, tty_name, sizeof(tty_name))); + ASSERT_STREQ(tty_name, name); + + struct winsize w_actual; + ASSERT_EQ(0, ioctl(slave, TIOCGWINSZ, &w_actual)); + ASSERT_EQ(w_actual.ws_row, w.ws_row); + ASSERT_EQ(w_actual.ws_col, w.ws_col); + ASSERT_EQ(w_actual.ws_xpixel, w.ws_xpixel); + ASSERT_EQ(w_actual.ws_ypixel, w.ws_ypixel); + + close(master); + close(slave); +} + +TEST(pty, forkpty) { + pid_t sid = getsid(0); + + int master; + pid_t pid = forkpty(&master, NULL, NULL, NULL); + ASSERT_NE(-1, pid); + + if (pid == 0) { + // We're the child. + ASSERT_NE(sid, getsid(0)); + _exit(0); + } + + ASSERT_EQ(sid, getsid(0)); + + int status; + ASSERT_EQ(pid, waitpid(pid, &status, 0)); + ASSERT_TRUE(WIFEXITED(status)); + ASSERT_EQ(0, WEXITSTATUS(status)); + + close(master); +} diff --git a/tests/utmp_test.cpp b/tests/utmp_test.cpp new file mode 100644 index 000000000..b61110d68 --- /dev/null +++ b/tests/utmp_test.cpp @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +TEST(utmp, login_tty) { + // login_tty is tested indirectly by the openpty and forkpty tests. + // This test just checks that we're exporting the symbol independently. + ASSERT_EQ(-1, login_tty(-1)); +}