From 0caf70e98e8cf109d0116c774147c568b369fd27 Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Wed, 23 Jul 2014 11:38:38 -0700 Subject: [PATCH 01/89] Fix belated review comments on syslog change. Bug: 14292866 (cherry picked from commit afe6360627ef3f0e9bc8f45535fbfae3354f3ae0) Change-Id: I8e3cc6b37b2539e51a27261ffb5d6e58266ce11d --- libc/bionic/libc_logging.cpp | 3 +- libc/bionic/syslog.cpp | 6 +-- libc/include/syslog.h | 77 +++++++++++++++++++----------------- 3 files changed, 45 insertions(+), 41 deletions(-) diff --git a/libc/bionic/libc_logging.cpp b/libc/bionic/libc_logging.cpp index 88d979091..cdbf7236c 100644 --- a/libc/bionic/libc_logging.cpp +++ b/libc/bionic/libc_logging.cpp @@ -230,6 +230,7 @@ static void SendRepeat(Out& o, char ch, int count) { /* Perform formatted output to an output target 'o' */ template static void out_vformat(Out& o, const char* format, va_list args) { + int caller_errno = errno; int nn = 0; for (;;) { @@ -380,7 +381,7 @@ static void out_vformat(Out& o, const char* format, va_list args) { buffer[1] = '\0'; } else if (c == 'm') { // syslog-like %m for strerror(errno). - str = strerror(errno); + str = strerror(caller_errno); } else { __assert(__FILE__, __LINE__, "conversion specifier unsupported"); } diff --git a/libc/bionic/syslog.cpp b/libc/bionic/syslog.cpp index 7e153eb16..29f892a20 100644 --- a/libc/bionic/syslog.cpp +++ b/libc/bionic/syslog.cpp @@ -14,9 +14,8 @@ * limitations under the License. */ -#include - #include +#include #include "private/libc_logging.h" @@ -24,6 +23,7 @@ static const char* syslog_log_tag = NULL; static int syslog_priority_mask = 0xff; void closelog() { + syslog_log_tag = NULL; } void openlog(const char* log_tag, int /*options*/, int /*facility*/) { @@ -61,7 +61,7 @@ void vsyslog(int priority, const char* fmt, va_list args) { // What's our Android log priority? priority &= LOG_PRIMASK; int android_log_priority; - if (priority < LOG_ERR) { + if (priority <= LOG_ERR) { android_log_priority = ANDROID_LOG_ERROR; } else if (priority == LOG_WARNING) { android_log_priority = ANDROID_LOG_WARN; diff --git a/libc/include/syslog.h b/libc/include/syslog.h index f396feca7..cbceab287 100644 --- a/libc/include/syslog.h +++ b/libc/include/syslog.h @@ -35,45 +35,48 @@ __BEGIN_DECLS -#define LOG_EMERG 0 -#define LOG_ALERT 1 -#define LOG_CRIT 2 -#define LOG_ERR 3 -#define LOG_WARNING 4 -#define LOG_NOTICE 5 -#define LOG_INFO 6 -#define LOG_DEBUG 7 +/* Priorities are translated to Android log priorities as shown. */ +#define LOG_EMERG 0 /* ERROR */ +#define LOG_ALERT 1 /* ERROR */ +#define LOG_CRIT 2 /* ERROR */ +#define LOG_ERR 3 /* ERROR */ +#define LOG_WARNING 4 /* WARN */ +#define LOG_NOTICE 5 /* INFO */ +#define LOG_INFO 6 /* INFO */ +#define LOG_DEBUG 7 /* DEBUG */ -#define LOG_PRIMASK 7 -#define LOG_PRI(x) ((x) & LOG_PRIMASK) +#define LOG_PRIMASK 7 +#define LOG_PRI(x) ((x) & LOG_PRIMASK) -#define LOG_KERN 0000 -#define LOG_USER 0010 -#define LOG_MAIL 0020 -#define LOG_DAEMON 0030 -#define LOG_AUTH 0040 -#define LOG_SYSLOG 0050 -#define LOG_LPR 0060 -#define LOG_NEWS 0070 -#define LOG_UUCP 0100 -#define LOG_CRON 0110 -#define LOG_AUTHPRIV 0120 -#define LOG_FTP 0130 -#define LOG_LOCAL0 0200 -#define LOG_LOCAL1 0210 -#define LOG_LOCAL2 0220 -#define LOG_LOCAL3 0230 -#define LOG_LOCAL4 0240 -#define LOG_LOCAL5 0250 -#define LOG_LOCAL6 0260 -#define LOG_LOCAL7 0270 +/* Facilities are currently ignored on Android. */ +#define LOG_KERN 0000 +#define LOG_USER 0010 +#define LOG_MAIL 0020 +#define LOG_DAEMON 0030 +#define LOG_AUTH 0040 +#define LOG_SYSLOG 0050 +#define LOG_LPR 0060 +#define LOG_NEWS 0070 +#define LOG_UUCP 0100 +#define LOG_CRON 0110 +#define LOG_AUTHPRIV 0120 +#define LOG_FTP 0130 +#define LOG_LOCAL0 0200 +#define LOG_LOCAL1 0210 +#define LOG_LOCAL2 0220 +#define LOG_LOCAL3 0230 +#define LOG_LOCAL4 0240 +#define LOG_LOCAL5 0250 +#define LOG_LOCAL6 0260 +#define LOG_LOCAL7 0270 -#define LOG_FACMASK 01770 -#define LOG_FAC(x) (((x) >> 3) & (LOG_FACMASK >> 3)) +#define LOG_FACMASK 01770 +#define LOG_FAC(x) (((x) >> 3) & (LOG_FACMASK >> 3)) #define LOG_MASK(pri) (1 << (pri)) #define LOG_UPTO(pri) ((1 << ((pri)+1)) - 1) +/* openlog(3) flags are currently ignored on Android. */ #define LOG_PID 0x01 #define LOG_CONS 0x02 #define LOG_ODELAY 0x04 @@ -81,11 +84,11 @@ __BEGIN_DECLS #define LOG_NOWAIT 0x10 #define LOG_PERROR 0x20 -extern void closelog(void); -extern void openlog(const char*, int, int); -extern int setlogmask(int); -extern void syslog(int, const char*, ...) __printflike(2, 3); -extern void vsyslog(int, const char*, va_list) __printflike(2, 0); +void closelog(void); +void openlog(const char*, int, int); +int setlogmask(int); +void syslog(int, const char*, ...) __printflike(2, 3); +void vsyslog(int, const char*, va_list) __printflike(2, 0); __END_DECLS From 4514aa630c8cace57a71268eabff687fcdf13ca0 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 23 Jul 2014 14:43:44 -0700 Subject: [PATCH 02/89] HACK: remove %m support from printf. The change that added this support causes a cpu hard lock on one device. This code clearly isn't at fault, but disabling it to unblock until we can find a real fix. Bug: 16484311 Change-Id: I33834dc49d959ae403b10d2c7cad12ae2950f772 --- libc/bionic/libc_logging.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libc/bionic/libc_logging.cpp b/libc/bionic/libc_logging.cpp index cdbf7236c..1fb8a84ab 100644 --- a/libc/bionic/libc_logging.cpp +++ b/libc/bionic/libc_logging.cpp @@ -230,7 +230,9 @@ static void SendRepeat(Out& o, char ch, int count) { /* Perform formatted output to an output target 'o' */ template static void out_vformat(Out& o, const char* format, va_list args) { +#if 0 int caller_errno = errno; +#endif int nn = 0; for (;;) { @@ -379,9 +381,11 @@ static void out_vformat(Out& o, const char* format, va_list args) { } else if (c == '%') { buffer[0] = '%'; buffer[1] = '\0'; +#if 0 } else if (c == 'm') { // syslog-like %m for strerror(errno). str = strerror(caller_errno); +#endif } else { __assert(__FILE__, __LINE__, "conversion specifier unsupported"); } From 11837629693e520452672e0eae28d3ce71f80ed6 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Thu, 24 Jul 2014 17:52:23 -0700 Subject: [PATCH 03/89] Use libunwindbacktrace for debug malloc code. Create a method of disabling the debug allocation code paths so that it's possible to use the libunwindbacktrace library without any modifications. Use this path to create and destroy the maps for the process. It's not stricly necessary in the init code since the symbols are not modified until after the initialize calls. Also, remove the debug_XXX source files that doesn't need to be in libc.so. Fix the maps reading code since it was completely broken for 64 bit. Bug: 16408686 (cherry picked from commit 861c0ef37bcfcae56d88572cb01c18bcfe1faded) Change-Id: I04445f0cf9a1e85172b64d57df92eb7939ce2332 --- libc/Android.mk | 8 ++-- libc/bionic/debug_mapinfo.cpp | 53 +++++++++++++++---------- libc/bionic/debug_mapinfo.h | 4 +- libc/bionic/debug_stacktrace.cpp | 13 +++++- libc/bionic/malloc_debug_check.cpp | 43 ++++++++++++++++++++ libc/bionic/malloc_debug_disable.h | 64 ++++++++++++++++++++++++++++++ libc/bionic/malloc_debug_leak.cpp | 41 +++++++++++++++++++ 7 files changed, 198 insertions(+), 28 deletions(-) create mode 100644 libc/bionic/malloc_debug_disable.h diff --git a/libc/Android.mk b/libc/Android.mk index 1fb5e84ae..8326dbbc8 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -982,8 +982,6 @@ LOCAL_SRC_FILES := \ $(libc_arch_dynamic_src_files) \ $(libc_static_common_src_files) \ bionic/malloc_debug_common.cpp \ - bionic/debug_mapinfo.cpp \ - bionic/debug_stacktrace.cpp \ bionic/libc_init_dynamic.cpp \ bionic/NetdClient.cpp \ @@ -1049,7 +1047,10 @@ LOCAL_CFLAGS := \ LOCAL_CONLYFLAGS := $(libc_common_conlyflags) LOCAL_CPPFLAGS := $(libc_common_cppflags) -LOCAL_C_INCLUDES := $(libc_common_c_includes) +# Make sure that unwind.h comes from libunwind. +LOCAL_C_INCLUDES := \ + external/libunwind/include \ + $(libc_common_c_includes) \ LOCAL_SRC_FILES := \ bionic/debug_mapinfo.cpp \ @@ -1064,6 +1065,7 @@ LOCAL_ADDITIONAL_DEPENDENCIES := $(libc_common_additional_dependencies) LOCAL_SHARED_LIBRARIES := libc libdl LOCAL_SYSTEM_SHARED_LIBRARIES := +LOCAL_WHOLE_STATIC_LIBRARIES := libunwindbacktrace LOCAL_ALLOW_UNDEFINED_SYMBOLS := true # Don't install on release build diff --git a/libc/bionic/debug_mapinfo.cpp b/libc/bionic/debug_mapinfo.cpp index 17276ce40..d83799afa 100644 --- a/libc/bionic/debug_mapinfo.cpp +++ b/libc/bionic/debug_mapinfo.cpp @@ -26,39 +26,48 @@ * SUCH DAMAGE. */ +#include +#include #include #include #include -#include #include "debug_mapinfo.h" +#include "malloc_debug_disable.h" -// 6f000000-6f01e000 rwxp 00000000 00:0c 16389419 /system/lib/libcomposer.so -// 012345678901234567890123456789012345678901234567890123456789 -// 0 1 2 3 4 5 - +// Format of /proc//maps: +// 6f000000-6f01e000 rwxp 00000000 00:0c 16389419 /system/lib/libcomposer.so static mapinfo_t* parse_maps_line(char* line) { - int len = strlen(line); - - if (len < 1) return 0; - line[--len] = 0; - - if (len < 50) return 0; - if (line[20] != 'x') return 0; - - mapinfo_t* mi = static_cast( - mmap(NULL, sizeof(mapinfo_t) + (len - 47), PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0)); - if (mi == MAP_FAILED) return 0; + uintptr_t start; + uintptr_t end; + int name_pos; + if (sscanf(line, "%" PRIxPTR "-%" PRIxPTR " %*4s %*x %*x:%*x %*d%n", &start, + &end, &name_pos) < 2) { + return NULL; + } - mi->start = strtoul(line, 0, 16); - mi->end = strtoul(line + 9, 0, 16); - mi->next = 0; - strcpy(mi->name, line + 49); + while (isspace(line[name_pos])) { + name_pos += 1; + } + const char* name = line + name_pos; + size_t name_len = strlen(name); + if (name_len && name[name_len - 1] == '\n') { + name_len -= 1; + } + mapinfo_t* mi = reinterpret_cast(calloc(1, sizeof(mapinfo_t) + name_len + 1)); + if (mi) { + mi->start = start; + mi->end = end; + memcpy(mi->name, name, name_len); + mi->name[name_len] = '\0'; + } return mi; } __LIBC_HIDDEN__ mapinfo_t* mapinfo_create(pid_t pid) { + ScopedDisableDebugCalls disable; + struct mapinfo_t* milist = NULL; char data[1024]; // Used to read lines as well as to construct the filename. snprintf(data, sizeof(data), "/proc/%d/maps", pid); @@ -77,10 +86,12 @@ __LIBC_HIDDEN__ mapinfo_t* mapinfo_create(pid_t pid) { } __LIBC_HIDDEN__ void mapinfo_destroy(mapinfo_t* mi) { + ScopedDisableDebugCalls disable; + while (mi != NULL) { mapinfo_t* del = mi; mi = mi->next; - munmap(del, sizeof(mapinfo_t) + strlen(del->name) + 2); + free(del); } } diff --git a/libc/bionic/debug_mapinfo.h b/libc/bionic/debug_mapinfo.h index cccd2e374..926b37762 100644 --- a/libc/bionic/debug_mapinfo.h +++ b/libc/bionic/debug_mapinfo.h @@ -33,8 +33,8 @@ struct mapinfo_t { struct mapinfo_t* next; - unsigned start; - unsigned end; + uintptr_t start; + uintptr_t end; char name[]; }; diff --git a/libc/bionic/debug_stacktrace.cpp b/libc/bionic/debug_stacktrace.cpp index 713e76113..b86e2afdc 100644 --- a/libc/bionic/debug_stacktrace.cpp +++ b/libc/bionic/debug_stacktrace.cpp @@ -35,6 +35,7 @@ #include #include "debug_mapinfo.h" +#include "malloc_debug_disable.h" #include "private/libc_logging.h" #if defined(__LP64__) @@ -56,6 +57,8 @@ typedef char* (*DemanglerFn)(const char*, char*, size_t*, int*); static DemanglerFn g_demangler_fn = NULL; __LIBC_HIDDEN__ void backtrace_startup() { + ScopedDisableDebugCalls disable; + g_map_info = mapinfo_create(getpid()); g_demangler = dlopen("libgccdemangle.so", RTLD_NOW); if (g_demangler != NULL) { @@ -65,6 +68,8 @@ __LIBC_HIDDEN__ void backtrace_startup() { } __LIBC_HIDDEN__ void backtrace_shutdown() { + ScopedDisableDebugCalls disable; + mapinfo_destroy(g_map_info); dlclose(g_demangler); } @@ -98,7 +103,7 @@ static _Unwind_Reason_Code trace_function(__unwind_context* context, void* arg) return _URC_NO_REASON; } -#ifdef __arm__ +#if defined(__arm__) /* * The instruction pointer is pointing at the instruction after the bl(x), and * the _Unwind_Backtrace routine already masks the Thumb mode indicator (LSB @@ -121,12 +126,16 @@ static _Unwind_Reason_Code trace_function(__unwind_context* context, void* arg) } __LIBC_HIDDEN__ int get_backtrace(uintptr_t* frames, size_t max_depth) { + ScopedDisableDebugCalls disable; + stack_crawl_state_t state(frames, max_depth); _Unwind_Backtrace(trace_function, &state); return state.frame_count; } __LIBC_HIDDEN__ void log_backtrace(uintptr_t* frames, size_t frame_count) { + ScopedDisableDebugCalls disable; + uintptr_t self_bt[16]; if (frames == NULL) { frame_count = get_backtrace(self_bt, 16); @@ -146,7 +155,7 @@ __LIBC_HIDDEN__ void log_backtrace(uintptr_t* frames, size_t frame_count) { symbol = info.dli_sname; } - uintptr_t rel_pc; + uintptr_t rel_pc = offset; const mapinfo_t* mi = (g_map_info != NULL) ? mapinfo_find(g_map_info, frames[i], &rel_pc) : NULL; const char* soname = (mi != NULL) ? mi->name : info.dli_fname; if (soname == NULL) { diff --git a/libc/bionic/malloc_debug_check.cpp b/libc/bionic/malloc_debug_check.cpp index 1c63d4dcf..94ba6f5ec 100644 --- a/libc/bionic/malloc_debug_check.cpp +++ b/libc/bionic/malloc_debug_check.cpp @@ -49,6 +49,7 @@ #include "debug_mapinfo.h" #include "debug_stacktrace.h" #include "malloc_debug_common.h" +#include "malloc_debug_disable.h" #include "private/bionic_macros.h" #include "private/libc_logging.h" #include "private/ScopedPthreadMutexLocker.h" @@ -331,6 +332,9 @@ static inline void add_to_backlog(hdr_t* hdr) { extern "C" void* chk_malloc(size_t bytes) { // log_message("%s: %s\n", __FILE__, __FUNCTION__); + if (DebugCallsDisabled()) { + return g_malloc_dispatch->malloc(bytes); + } size_t size = sizeof(hdr_t) + bytes + sizeof(ftr_t); if (size < bytes) { // Overflow @@ -348,6 +352,10 @@ extern "C" void* chk_malloc(size_t bytes) { } extern "C" void* chk_memalign(size_t alignment, size_t bytes) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->memalign(alignment, bytes); + } + if (alignment <= MALLOC_ALIGNMENT) { return chk_malloc(bytes); } @@ -386,6 +394,9 @@ extern "C" void* chk_memalign(size_t alignment, size_t bytes) { extern "C" void chk_free(void* ptr) { // log_message("%s: %s\n", __FILE__, __FUNCTION__); + if (DebugCallsDisabled()) { + return g_malloc_dispatch->free(ptr); + } if (!ptr) /* ignore free(NULL) */ return; @@ -421,6 +432,9 @@ extern "C" void chk_free(void* ptr) { extern "C" void* chk_realloc(void* ptr, size_t bytes) { // log_message("%s: %s\n", __FILE__, __FUNCTION__); + if (DebugCallsDisabled()) { + return g_malloc_dispatch->realloc(ptr, bytes); + } if (!ptr) { return chk_malloc(bytes); @@ -496,6 +510,10 @@ extern "C" void* chk_realloc(void* ptr, size_t bytes) { extern "C" void* chk_calloc(size_t nmemb, size_t bytes) { // log_message("%s: %s\n", __FILE__, __FUNCTION__); + if (DebugCallsDisabled()) { + return g_malloc_dispatch->calloc(nmemb, bytes); + } + size_t total_bytes = nmemb * bytes; size_t size = sizeof(hdr_t) + total_bytes + sizeof(ftr_t); if (size < total_bytes || (nmemb && SIZE_MAX / nmemb < bytes)) { // Overflow @@ -513,6 +531,10 @@ extern "C" void* chk_calloc(size_t nmemb, size_t bytes) { } extern "C" size_t chk_malloc_usable_size(const void* ptr) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->malloc_usable_size(ptr); + } + // malloc_usable_size returns 0 for NULL and unknown blocks. if (ptr == NULL) return 0; @@ -529,6 +551,10 @@ extern "C" struct mallinfo chk_mallinfo() { } extern "C" int chk_posix_memalign(void** memptr, size_t alignment, size_t size) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->posix_memalign(memptr, alignment, size); + } + if (!powerof2(alignment)) { return EINVAL; } @@ -538,7 +564,12 @@ extern "C" int chk_posix_memalign(void** memptr, size_t alignment, size_t size) return (*memptr != NULL) ? 0 : ENOMEM; } +#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) extern "C" void* chk_pvalloc(size_t bytes) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->pvalloc(bytes); + } + size_t pagesize = getpagesize(); size_t size = BIONIC_ALIGN(bytes, pagesize); if (size < bytes) { // Overflow @@ -548,10 +579,16 @@ extern "C" void* chk_pvalloc(size_t bytes) { } extern "C" void* chk_valloc(size_t size) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->valloc(size); + } return chk_memalign(getpagesize(), size); } +#endif static void ReportMemoryLeaks() { + ScopedDisableDebugCalls disable; + // Use /proc/self/exe link to obtain the program name for logging // purposes. If it's not available, we set it to "". char exe[PATH_MAX]; @@ -585,10 +622,14 @@ static void ReportMemoryLeaks() { } } +pthread_key_t g_debug_calls_disabled; + extern "C" bool malloc_debug_initialize(HashTable* hash_table, const MallocDebug* malloc_dispatch) { g_hash_table = hash_table; g_malloc_dispatch = malloc_dispatch; + pthread_key_create(&g_debug_calls_disabled, NULL); + char debug_backlog[PROP_VALUE_MAX]; if (__system_property_get("libc.debug.malloc.backlog", debug_backlog)) { g_malloc_debug_backlog = atoi(debug_backlog); @@ -605,4 +646,6 @@ extern "C" void malloc_debug_finalize(int malloc_debug_level) { ReportMemoryLeaks(); } backtrace_shutdown(); + + pthread_setspecific(g_debug_calls_disabled, NULL); } diff --git a/libc/bionic/malloc_debug_disable.h b/libc/bionic/malloc_debug_disable.h new file mode 100644 index 000000000..9503128c4 --- /dev/null +++ b/libc/bionic/malloc_debug_disable.h @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef MALLOC_DEBUG_DISABLE_H +#define MALLOC_DEBUG_DISABLE_H + +#include + +#include "private/bionic_macros.h" + +// ============================================================================= +// Used to disable the debug allocation calls. +// ============================================================================= +extern pthread_key_t g_debug_calls_disabled; + +static inline bool DebugCallsDisabled() { + return pthread_getspecific(g_debug_calls_disabled) != NULL; +} + +class ScopedDisableDebugCalls { + public: + ScopedDisableDebugCalls() : disabled_(DebugCallsDisabled()) { + if (!disabled_) { + pthread_setspecific(g_debug_calls_disabled, reinterpret_cast(1)); + } + } + ~ScopedDisableDebugCalls() { + if (!disabled_) { + pthread_setspecific(g_debug_calls_disabled, NULL); + } + } + + private: + bool disabled_; + + DISALLOW_COPY_AND_ASSIGN(ScopedDisableDebugCalls); +}; + +#endif // MALLOC_DEBUG_DISABLE_H diff --git a/libc/bionic/malloc_debug_leak.cpp b/libc/bionic/malloc_debug_leak.cpp index d9824f09f..7926a1f27 100644 --- a/libc/bionic/malloc_debug_leak.cpp +++ b/libc/bionic/malloc_debug_leak.cpp @@ -48,6 +48,7 @@ #include "debug_stacktrace.h" #include "malloc_debug_common.h" +#include "malloc_debug_disable.h" #include "private/bionic_macros.h" #include "private/libc_logging.h" @@ -267,6 +268,7 @@ extern "C" int fill_posix_memalign(void** memptr, size_t alignment, size_t size) return (*memptr != NULL) ? 0 : ENOMEM; } +#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) extern "C" void* fill_pvalloc(size_t bytes) { size_t pagesize = getpagesize(); size_t size = BIONIC_ALIGN(bytes, pagesize); @@ -279,6 +281,7 @@ extern "C" void* fill_pvalloc(size_t bytes) { extern "C" void* fill_valloc(size_t size) { return fill_memalign(getpagesize(), size); } +#endif // ============================================================================= // malloc leak functions @@ -287,6 +290,10 @@ extern "C" void* fill_valloc(size_t size) { static uint32_t MEMALIGN_GUARD = 0xA1A41520; extern "C" void* leak_malloc(size_t bytes) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->malloc(bytes); + } + // allocate enough space infront of the allocation to store the pointer for // the alloc structure. This will making free'ing the structer really fast! @@ -319,6 +326,10 @@ extern "C" void* leak_malloc(size_t bytes) { } extern "C" void leak_free(void* mem) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->free(mem); + } + if (mem == NULL) { return; } @@ -355,6 +366,10 @@ extern "C" void leak_free(void* mem) { } extern "C" void* leak_calloc(size_t n_elements, size_t elem_size) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->calloc(n_elements, elem_size); + } + // Fail on overflow - just to be safe even though this code runs only // within the debugging C library, not the production one. if (n_elements && SIZE_MAX / n_elements < elem_size) { @@ -370,6 +385,10 @@ extern "C" void* leak_calloc(size_t n_elements, size_t elem_size) { } extern "C" void* leak_realloc(void* oldMem, size_t bytes) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->realloc(oldMem, bytes); + } + if (oldMem == NULL) { return leak_malloc(bytes); } @@ -398,6 +417,10 @@ extern "C" void* leak_realloc(void* oldMem, size_t bytes) { } extern "C" void* leak_memalign(size_t alignment, size_t bytes) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->memalign(alignment, bytes); + } + // we can just use malloc if (alignment <= MALLOC_ALIGNMENT) { return leak_malloc(bytes); @@ -439,6 +462,10 @@ extern "C" void* leak_memalign(size_t alignment, size_t bytes) { } extern "C" size_t leak_malloc_usable_size(const void* mem) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->malloc_usable_size(mem); + } + if (mem != NULL) { // Check the guard to make sure it is valid. const AllocationEntry* header = const_to_header((void*)mem); @@ -467,6 +494,10 @@ extern "C" struct mallinfo leak_mallinfo() { } extern "C" int leak_posix_memalign(void** memptr, size_t alignment, size_t size) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->posix_memalign(memptr, alignment, size); + } + if (!powerof2(alignment)) { return EINVAL; } @@ -476,7 +507,12 @@ extern "C" int leak_posix_memalign(void** memptr, size_t alignment, size_t size) return (*memptr != NULL) ? 0 : ENOMEM; } +#if defined(HAVE_DEPRECATED_MALLOC_FUNCS) extern "C" void* leak_pvalloc(size_t bytes) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->pvalloc(bytes); + } + size_t pagesize = getpagesize(); size_t size = BIONIC_ALIGN(bytes, pagesize); if (size < bytes) { // Overflow @@ -486,5 +522,10 @@ extern "C" void* leak_pvalloc(size_t bytes) { } extern "C" void* leak_valloc(size_t size) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->valloc(size); + } + return leak_memalign(getpagesize(), size); } +#endif From b5e08542840d5722defae3e750d49a7d5ce6ccc9 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Thu, 7 Aug 2014 16:21:21 -0700 Subject: [PATCH 04/89] Add a way to disable backtracing in malloc debug. The property libc.debug.malloc.nobacktrace set to non-zero disables getting backtracing when using mode 1 or mode 10. Bug: 16874447 Change-Id: I7650ba9f4385b5110b743cab01e877fc69545b3c --- libc/bionic/malloc_debug_backtrace.h | 37 +++++++++++ libc/bionic/malloc_debug_check.cpp | 95 ++++++++++++++++++---------- libc/bionic/malloc_debug_leak.cpp | 3 +- 3 files changed, 99 insertions(+), 36 deletions(-) create mode 100644 libc/bionic/malloc_debug_backtrace.h diff --git a/libc/bionic/malloc_debug_backtrace.h b/libc/bionic/malloc_debug_backtrace.h new file mode 100644 index 000000000..774548b59 --- /dev/null +++ b/libc/bionic/malloc_debug_backtrace.h @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef MALLOC_DEBUG_BACKTRACE_H +#define MALLOC_DEBUG_BACKTRACE_H + +extern bool g_backtrace_enabled; + +#define GET_BACKTRACE(bt, depth) \ + (g_backtrace_enabled ? get_backtrace(bt, depth) : 0) + +#endif // MALLOC_DEBUG_BACKTRACE_H diff --git a/libc/bionic/malloc_debug_check.cpp b/libc/bionic/malloc_debug_check.cpp index 94ba6f5ec..dee03fae1 100644 --- a/libc/bionic/malloc_debug_check.cpp +++ b/libc/bionic/malloc_debug_check.cpp @@ -48,6 +48,7 @@ #include "debug_mapinfo.h" #include "debug_stacktrace.h" +#include "malloc_debug_backtrace.h" #include "malloc_debug_common.h" #include "malloc_debug_disable.h" #include "private/bionic_macros.h" @@ -123,6 +124,10 @@ static pthread_mutex_t backlog_lock = PTHREAD_MUTEX_INITIALIZER; // It determines the size of the backlog we use to detect multiple frees. static unsigned g_malloc_debug_backlog = 100; +// This variable is set to false if the property libc.debug.malloc.nobacktrace +// is set to non-zero. +__LIBC_HIDDEN__ bool g_backtrace_enabled = true; + __LIBC_HIDDEN__ HashTable* g_hash_table; __LIBC_HIDDEN__ const MallocDebug* g_malloc_dispatch; @@ -273,7 +278,7 @@ static inline int check_allocation_locked(hdr_t* hdr, int* safe) { valid = check_guards(hdr, safe); } - if (!valid && *safe) { + if (!valid && *safe && g_backtrace_enabled) { log_message("+++ ALLOCATION %p SIZE %d ALLOCATED HERE:\n", user(hdr), hdr->size); log_backtrace(hdr->bt, hdr->bt_depth); @@ -344,7 +349,7 @@ extern "C" void* chk_malloc(size_t bytes) { hdr_t* hdr = static_cast(g_malloc_dispatch->malloc(size)); if (hdr) { hdr->base = hdr; - hdr->bt_depth = get_backtrace(hdr->bt, MAX_BACKTRACE_DEPTH); + hdr->bt_depth = GET_BACKTRACE(hdr->bt, MAX_BACKTRACE_DEPTH); add(hdr, bytes); return user(hdr); } @@ -385,7 +390,7 @@ extern "C" void* chk_memalign(size_t alignment, size_t bytes) { hdr_t* hdr = meta(reinterpret_cast(ptr)); hdr->base = base; - hdr->bt_depth = get_backtrace(hdr->bt, MAX_BACKTRACE_DEPTH); + hdr->bt_depth = GET_BACKTRACE(hdr->bt, MAX_BACKTRACE_DEPTH); add(hdr, bytes); return user(hdr); } @@ -405,27 +410,31 @@ extern "C" void chk_free(void* ptr) { if (del(hdr) < 0) { uintptr_t bt[MAX_BACKTRACE_DEPTH]; - int depth = get_backtrace(bt, MAX_BACKTRACE_DEPTH); + int depth = GET_BACKTRACE(bt, MAX_BACKTRACE_DEPTH); if (hdr->tag == BACKLOG_TAG) { log_message("+++ ALLOCATION %p SIZE %d BYTES MULTIPLY FREED!\n", user(hdr), hdr->size); - log_message("+++ ALLOCATION %p SIZE %d ALLOCATED HERE:\n", - user(hdr), hdr->size); - log_backtrace(hdr->bt, hdr->bt_depth); - /* hdr->freed_bt_depth should be nonzero here */ - log_message("+++ ALLOCATION %p SIZE %d FIRST FREED HERE:\n", - user(hdr), hdr->size); - log_backtrace(hdr->freed_bt, hdr->freed_bt_depth); - log_message("+++ ALLOCATION %p SIZE %d NOW BEING FREED HERE:\n", - user(hdr), hdr->size); - log_backtrace(bt, depth); + if (g_backtrace_enabled) { + log_message("+++ ALLOCATION %p SIZE %d ALLOCATED HERE:\n", + user(hdr), hdr->size); + log_backtrace(hdr->bt, hdr->bt_depth); + /* hdr->freed_bt_depth should be nonzero here */ + log_message("+++ ALLOCATION %p SIZE %d FIRST FREED HERE:\n", + user(hdr), hdr->size); + log_backtrace(hdr->freed_bt, hdr->freed_bt_depth); + log_message("+++ ALLOCATION %p SIZE %d NOW BEING FREED HERE:\n", + user(hdr), hdr->size); + log_backtrace(bt, depth); + } } else { log_message("+++ ALLOCATION %p IS CORRUPTED OR NOT ALLOCATED VIA TRACKER!\n", user(hdr)); - log_backtrace(bt, depth); + if (g_backtrace_enabled) { + log_backtrace(bt, depth); + } } } else { - hdr->freed_bt_depth = get_backtrace(hdr->freed_bt, MAX_BACKTRACE_DEPTH); + hdr->freed_bt_depth = GET_BACKTRACE(hdr->freed_bt, MAX_BACKTRACE_DEPTH); add_to_backlog(hdr); } } @@ -451,22 +460,24 @@ extern "C" void* chk_realloc(void* ptr, size_t bytes) { if (del(hdr) < 0) { uintptr_t bt[MAX_BACKTRACE_DEPTH]; - int depth = get_backtrace(bt, MAX_BACKTRACE_DEPTH); + int depth = GET_BACKTRACE(bt, MAX_BACKTRACE_DEPTH); if (hdr->tag == BACKLOG_TAG) { log_message("+++ REALLOCATION %p SIZE %d OF FREED MEMORY!\n", user(hdr), bytes, hdr->size); - log_message("+++ ALLOCATION %p SIZE %d ALLOCATED HERE:\n", - user(hdr), hdr->size); - log_backtrace(hdr->bt, hdr->bt_depth); - /* hdr->freed_bt_depth should be nonzero here */ - log_message("+++ ALLOCATION %p SIZE %d FIRST FREED HERE:\n", - user(hdr), hdr->size); - log_backtrace(hdr->freed_bt, hdr->freed_bt_depth); - log_message("+++ ALLOCATION %p SIZE %d NOW BEING REALLOCATED HERE:\n", - user(hdr), hdr->size); - log_backtrace(bt, depth); - - /* We take the memory out of the backlog and fall through so the + if (g_backtrace_enabled) { + log_message("+++ ALLOCATION %p SIZE %d ALLOCATED HERE:\n", + user(hdr), hdr->size); + log_backtrace(hdr->bt, hdr->bt_depth); + /* hdr->freed_bt_depth should be nonzero here */ + log_message("+++ ALLOCATION %p SIZE %d FIRST FREED HERE:\n", + user(hdr), hdr->size); + log_backtrace(hdr->freed_bt, hdr->freed_bt_depth); + log_message("+++ ALLOCATION %p SIZE %d NOW BEING REALLOCATED HERE:\n", + user(hdr), hdr->size); + log_backtrace(bt, depth); + } + + /* We take the memory out of the backlog and fall through so the * reallocation below succeeds. Since we didn't really free it, we * can default to this behavior. */ @@ -474,7 +485,9 @@ extern "C" void* chk_realloc(void* ptr, size_t bytes) { } else { log_message("+++ REALLOCATION %p SIZE %d IS CORRUPTED OR NOT ALLOCATED VIA TRACKER!\n", user(hdr), bytes); - log_backtrace(bt, depth); + if (g_backtrace_enabled) { + log_backtrace(bt, depth); + } // just get a whole new allocation and leak the old one return g_malloc_dispatch->realloc(0, bytes); // return realloc(user(hdr), bytes); // assuming it was allocated externally @@ -501,7 +514,7 @@ extern "C" void* chk_realloc(void* ptr, size_t bytes) { } if (hdr) { hdr->base = hdr; - hdr->bt_depth = get_backtrace(hdr->bt, MAX_BACKTRACE_DEPTH); + hdr->bt_depth = GET_BACKTRACE(hdr->bt, MAX_BACKTRACE_DEPTH); add(hdr, bytes); return user(hdr); } @@ -523,7 +536,7 @@ extern "C" void* chk_calloc(size_t nmemb, size_t bytes) { hdr_t* hdr = static_cast(g_malloc_dispatch->calloc(1, size)); if (hdr) { hdr->base = hdr; - hdr->bt_depth = get_backtrace(hdr->bt, MAX_BACKTRACE_DEPTH); + hdr->bt_depth = GET_BACKTRACE(hdr->bt, MAX_BACKTRACE_DEPTH); add(hdr, total_bytes); return user(hdr); } @@ -611,7 +624,7 @@ static void ReportMemoryLeaks() { hdr_t* block = head; log_message("+++ %s leaked block of size %d at %p (leak %d of %d)", exe, block->size, user(block), index++, total); - if (del_leak(block, &safe)) { + if (del_leak(block, &safe) && g_backtrace_enabled) { /* safe == 1, because the allocation is valid */ log_backtrace(block->bt, block->bt_depth); } @@ -636,7 +649,17 @@ extern "C" bool malloc_debug_initialize(HashTable* hash_table, const MallocDebug info_log("%s: setting backlog length to %d\n", getprogname(), g_malloc_debug_backlog); } - backtrace_startup(); + // Check if backtracing should be disabled. + char env[PROP_VALUE_MAX]; + if (__system_property_get("libc.debug.malloc.nobacktrace", env) && atoi(env) != 0) { + g_backtrace_enabled = false; + __libc_format_log(ANDROID_LOG_INFO, "libc", "not gathering backtrace information\n"); + } + + if (g_backtrace_enabled) { + backtrace_startup(); + } + return true; } @@ -645,7 +668,9 @@ extern "C" void malloc_debug_finalize(int malloc_debug_level) { if (malloc_debug_level == 10) { ReportMemoryLeaks(); } - backtrace_shutdown(); + if (g_backtrace_enabled) { + backtrace_shutdown(); + } pthread_setspecific(g_debug_calls_disabled, NULL); } diff --git a/libc/bionic/malloc_debug_leak.cpp b/libc/bionic/malloc_debug_leak.cpp index 7926a1f27..837dccc5a 100644 --- a/libc/bionic/malloc_debug_leak.cpp +++ b/libc/bionic/malloc_debug_leak.cpp @@ -47,6 +47,7 @@ #include #include "debug_stacktrace.h" +#include "malloc_debug_backtrace.h" #include "malloc_debug_common.h" #include "malloc_debug_disable.h" @@ -311,7 +312,7 @@ extern "C" void* leak_malloc(size_t bytes) { ScopedPthreadMutexLocker locker(&g_hash_table->lock); uintptr_t backtrace[BACKTRACE_SIZE]; - size_t numEntries = get_backtrace(backtrace, BACKTRACE_SIZE); + size_t numEntries = GET_BACKTRACE(backtrace, BACKTRACE_SIZE); AllocationEntry* header = reinterpret_cast(base); header->entry = record_backtrace(backtrace, numEntries, bytes); From a0108accb212fd916fb776e6675cc6261d1b1433 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Fri, 15 Aug 2014 18:42:58 -0700 Subject: [PATCH 05/89] Fix leak_realloc, copy entire allocation. Bug: 16874447 Change-Id: Ie54a73fd75529961195fa5173d9116d0ae897b03 --- libc/bionic/malloc_debug_leak.cpp | 64 +++++++++++++++---------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/libc/bionic/malloc_debug_leak.cpp b/libc/bionic/malloc_debug_leak.cpp index 27b671432..df0f997e5 100644 --- a/libc/bionic/malloc_debug_leak.cpp +++ b/libc/bionic/malloc_debug_leak.cpp @@ -385,6 +385,36 @@ extern "C" void* leak_calloc(size_t n_elements, size_t elem_size) { return ptr; } +extern "C" size_t leak_malloc_usable_size(const void* mem) { + if (DebugCallsDisabled()) { + return g_malloc_dispatch->malloc_usable_size(mem); + } + + if (mem == NULL) { + return 0; + } + + // Check the guard to make sure it is valid. + const AllocationEntry* header = const_to_header(mem); + + if (header->guard == MEMALIGN_GUARD) { + // If this is a memalign'd pointer, then grab the header from + // entry. + header = const_to_header(header->entry); + } else if (header->guard != GUARD) { + debug_log("WARNING bad header guard: '0x%x'! and invalid entry: %p\n", + header->guard, header->entry); + return 0; + } + + size_t ret = g_malloc_dispatch->malloc_usable_size(header); + if (ret != 0) { + // The usable area starts at 'mem' and stops at 'header+ret'. + return reinterpret_cast(header) + ret - reinterpret_cast(mem); + } + return 0; +} + extern "C" void* leak_realloc(void* oldMem, size_t bytes) { if (DebugCallsDisabled()) { return g_malloc_dispatch->realloc(oldMem, bytes); @@ -408,7 +438,7 @@ extern "C" void* leak_realloc(void* oldMem, size_t bytes) { newMem = leak_malloc(bytes); if (newMem != NULL) { - size_t oldSize = header->entry->size & ~SIZE_FLAG_MASK; + size_t oldSize = leak_malloc_usable_size(oldMem); size_t copySize = (oldSize <= bytes) ? oldSize : bytes; memcpy(newMem, oldMem, copySize); leak_free(oldMem); @@ -462,38 +492,6 @@ extern "C" void* leak_memalign(size_t alignment, size_t bytes) { return base; } -extern "C" size_t leak_malloc_usable_size(const void* mem) { - if (DebugCallsDisabled()) { - return g_malloc_dispatch->malloc_usable_size(mem); - } - - if (mem != NULL) { - // Check the guard to make sure it is valid. - const AllocationEntry* header = const_to_header((void*)mem); - - if (header->guard == MEMALIGN_GUARD) { - // If this is a memalign'd pointer, then grab the header from - // entry. - header = const_to_header(header->entry); - } else if (header->guard != GUARD) { - debug_log("WARNING bad header guard: '0x%x'! and invalid entry: %p\n", - header->guard, header->entry); - return 0; - } - - // TODO: Temporary workaround to avoid a crash b/16874447. - return header->entry->size & ~SIZE_FLAG_MASK; -#if 0 - size_t ret = g_malloc_dispatch->malloc_usable_size(header); - if (ret != 0) { - // The usable area starts at 'mem' and stops at 'header+ret'. - return reinterpret_cast(header) + ret - reinterpret_cast(mem); - } -#endif - } - return 0; -} - extern "C" struct mallinfo leak_mallinfo() { return g_malloc_dispatch->mallinfo(); } From 20dc3f8fa4b192d902d58f496ae15ff33faa78ac Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Tue, 26 Aug 2014 20:48:11 -0700 Subject: [PATCH 06/89] Replace snprintf calls in linker. When enabling debug malloc, the snprintf calls in the linker fails to update the buffer. The problem is that snprintf makes a call to pthread_getspecific that returns a valid pointer, but the data it points to is zero. This should never happen and causes the snprintf to stop and do nothing. Temporarily replace snprintf with a different implementation to work around this issue. Bug: 16874447 Bug: 17302493 Change-Id: I7a500f28adf153150cf2812fae745ff41f1c48d3 --- linker/debugger.cpp | 4 ++-- linker/linker.cpp | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/linker/debugger.cpp b/linker/debugger.cpp index 079682cab..c31615171 100644 --- a/linker/debugger.cpp +++ b/linker/debugger.cpp @@ -170,9 +170,9 @@ static void log_signal_summary(int signum, const siginfo_t* info) { if (info != NULL) { // For a rethrown signal, this si_code will be right and the one debuggerd shows will // always be SI_TKILL. - snprintf(code_desc, sizeof(code_desc), ", code %d", info->si_code); + __libc_format_buffer(code_desc, sizeof(code_desc), ", code %d", info->si_code); if (has_address) { - snprintf(addr_desc, sizeof(addr_desc), ", fault addr %p", info->si_addr); + __libc_format_buffer(addr_desc, sizeof(addr_desc), ", fault addr %p", info->si_addr); } } __libc_format_log(ANDROID_LOG_FATAL, "libc", diff --git a/linker/linker.cpp b/linker/linker.cpp index 52eb56ab8..2d8e07eb4 100644 --- a/linker/linker.cpp +++ b/linker/linker.cpp @@ -866,7 +866,21 @@ static void soinfo_unload(soinfo* si) { } void do_android_get_LD_LIBRARY_PATH(char* buffer, size_t buffer_size) { - snprintf(buffer, buffer_size, "%s:%s", kDefaultLdPaths[0], kDefaultLdPaths[1]); + // Use basic string manipulation calls to avoid snprintf. + // snprintf indirectly calls pthread_getspecific to get the size of a buffer. + // When debug malloc is enabled, this call returns 0. This in turn causes + // snprintf to do nothing, which causes libraries to fail to load. + // See b/17302493 for further details. + // Once the above bug is fixed, this code can be modified to use + // snprintf again. + size_t required_len = strlen(kDefaultLdPaths[0]) + strlen(kDefaultLdPaths[1]) + 2; + if (buffer_size < required_len) { + __libc_fatal("android_get_LD_LIBRARY_PATH failed, buffer too small: buffer len %zu, required len %zu", + buffer_size, required_len); + } + char* end = stpcpy(buffer, kDefaultLdPaths[0]); + *end = ':'; + strcpy(end + 1, kDefaultLdPaths[1]); } void do_android_update_LD_LIBRARY_PATH(const char* ld_library_path) { From 24a365fbe49e21407247c2d696cf46cddf0dfd14 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Mon, 6 Oct 2014 15:11:28 -0700 Subject: [PATCH 07/89] Force export symbols on all x86 variants in libc. For silvermont, the __popcountsi2 symbol does not get exported by libc. But for atom, this symbol is exported. Since we already exported this symbol for previous releases, it's better to just follow through and force the export, but only for 32 bit. x86 64 bit will not export this symbol. Bug: 17681440 Change-Id: I6c62245f0960910f64baaaf6d9d090bf3ea5f435 --- libc/arch-x86/bionic/libgcc_compat.c | 15 +++++++++++++++ libc/arch-x86/x86.mk | 1 + libc/tools/genlibgcc_compat.py | 11 ++++++----- 3 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 libc/arch-x86/bionic/libgcc_compat.c diff --git a/libc/arch-x86/bionic/libgcc_compat.c b/libc/arch-x86/bionic/libgcc_compat.c new file mode 100644 index 000000000..c7232638f --- /dev/null +++ b/libc/arch-x86/bionic/libgcc_compat.c @@ -0,0 +1,15 @@ +/* Generated by genlibgcc_compat.py */ + +extern char __divdi3; +extern char __moddi3; +extern char __popcountsi2; +extern char __udivdi3; +extern char __umoddi3; + +void* __bionic_libgcc_compat_symbols[] = { + &__divdi3, + &__moddi3, + &__popcountsi2, + &__udivdi3, + &__umoddi3, +}; diff --git a/libc/arch-x86/x86.mk b/libc/arch-x86/x86.mk index a14154866..2a0609dde 100644 --- a/libc/arch-x86/x86.mk +++ b/libc/arch-x86/x86.mk @@ -25,6 +25,7 @@ libc_bionic_src_files_x86 := \ libc_bionic_src_files_x86 += \ arch-x86/bionic/__bionic_clone.S \ arch-x86/bionic/_exit_with_stack_teardown.S \ + arch-x86/bionic/libgcc_compat.c \ arch-x86/bionic/_setjmp.S \ arch-x86/bionic/setjmp.S \ arch-x86/bionic/__set_tls.c \ diff --git a/libc/tools/genlibgcc_compat.py b/libc/tools/genlibgcc_compat.py index cb5c3b38e..628bf9217 100755 --- a/libc/tools/genlibgcc_compat.py +++ b/libc/tools/genlibgcc_compat.py @@ -76,9 +76,6 @@ class Generator: def process(self): android_build_top_path = os.environ["ANDROID_BUILD_TOP"] - build_path = android_build_top_path + "/bionic/libc" - file_name = "libgcc_compat.c" - file_path = build_path + "/arch-arm/bionic/" + file_name print "* ANDROID_BUILD_TOP=" + android_build_top_path @@ -86,8 +83,12 @@ def process(self): arch = subprocess.check_output(["CALLED_FROM_SETUP=true BUILD_SYSTEM=build/core make --no-print-directory -f build/core/config.mk dumpvar-TARGET_ARCH"], cwd=android_build_top_path, shell=True).strip() - if arch != 'arm': - sys.exit("Error: Invalid TARGET_ARCH='" + arch + "' expecting 'arm'") + if arch != 'arm' and arch != 'x86': + sys.exit("Error: Invalid TARGET_ARCH='" + arch + "' expecting 'arm' or 'x86'") + + build_path = android_build_top_path + "/bionic/libc" + file_name = "libgcc_compat.c" + file_path = build_path + "/arch-" + arch + "/bionic/" + file_name build_output_file_path = tempfile.mkstemp()[1] From 2415f1feb1defde671dfc6e9276a58e9a3f447ed Mon Sep 17 00:00:00 2001 From: Chiou-Hao Hsu Date: Wed, 2 Jul 2014 18:46:28 -0700 Subject: [PATCH 08/89] Include asm/sigcontext.h only for LP64 in singnal.h Causes compilation issues reporting no definition of __uint128_t. Change-Id: I10db050c63fbff866671d53bfd4ce66d3ea7cc7f --- libc/include/signal.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libc/include/signal.h b/libc/include/signal.h index f1849c5d4..7d15edade 100644 --- a/libc/include/signal.h +++ b/libc/include/signal.h @@ -34,7 +34,9 @@ #include /* For LONG_BIT */ #include /* For memset() */ #include +#if defined(__LP64__) #include +#endif #if defined(__LP64__) || defined(__mips__) /* For 64-bit (and mips), the kernel's struct sigaction doesn't match the POSIX one, From a22316478d940e2d858f4de51bd4c8f3438f6fe3 Mon Sep 17 00:00:00 2001 From: Pracheer Date: Fri, 28 Mar 2014 11:32:50 +0530 Subject: [PATCH 09/89] libc: Add cortex-a53 bionic code Add cortex-a53 makefile. Currently it uses cortex-a15 bionic code. Change-Id: I1401b54b6c61d17782dff827d69f92b35a4f5c4f --- libc/arch-arm/cortex-a53/cortex-a53.mk | 1 + 1 file changed, 1 insertion(+) create mode 100644 libc/arch-arm/cortex-a53/cortex-a53.mk diff --git a/libc/arch-arm/cortex-a53/cortex-a53.mk b/libc/arch-arm/cortex-a53/cortex-a53.mk new file mode 100644 index 000000000..9af03d94b --- /dev/null +++ b/libc/arch-arm/cortex-a53/cortex-a53.mk @@ -0,0 +1 @@ +include bionic/libc/arch-arm/cortex-a15/cortex-a15.mk From e7632cebdc3e84c8e3a40f09d8d5496c4035808c Mon Sep 17 00:00:00 2001 From: Xin Qi Date: Fri, 15 Aug 2014 11:12:32 -0700 Subject: [PATCH 10/89] libc: krait: Implement optimized versions of memmove and bcopy Code has been refactored to thumb2 for consistency with the rest of bionic libc, as well as performance and correctness. Use memove & bcopy in cortex-a53. Change-Id: I5f738ef3eb12ece6b55285f1588eab3d4bbbe27d --- libc/arch-arm/arm.mk | 7 +- libc/arch-arm/cortex-a53/cortex-a53.mk | 11 +- libc/arch-arm/krait/bionic/memmove.S | 219 +++++++++++++++++++++++++ libc/arch-arm/krait/krait.mk | 3 +- 4 files changed, 237 insertions(+), 3 deletions(-) create mode 100644 libc/arch-arm/krait/bionic/memmove.S diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index 70cc8eba6..7d92ac89c 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -25,7 +25,6 @@ libc_common_src_files_arm += \ upstream-freebsd/lib/libc/string/wcsrchr.c \ upstream-freebsd/lib/libc/string/wmemcmp.c \ upstream-freebsd/lib/libc/string/wmemmove.c \ - upstream-openbsd/lib/libc/string/bcopy.c \ upstream-openbsd/lib/libc/string/stpcpy.c \ upstream-openbsd/lib/libc/string/stpncpy.c \ upstream-openbsd/lib/libc/string/strlcat.c \ @@ -70,6 +69,12 @@ libc_netbsd_src_files_arm := \ ifeq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),) $(warning TARGET_$(my_2nd_arch_prefix)ARCH is arm, but TARGET_$(my_2nd_arch_prefix)CPU_VARIANT is not defined) endif +ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),krait) +ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),cortex-a53) +libc_bionic_src_files_arm += \ + upstream-openbsd/lib/libc/string/bcopy.c +endif +endif cpu_variant_mk := $(LOCAL_PATH)/arch-arm/$(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)/$(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT).mk ifeq ($(wildcard $(cpu_variant_mk)),) $(error "TARGET_$(my_2nd_arch_prefix)CPU_VARIANT not set or set to an unknown value. Possible values are cortex-a7, cortex-a8, cortex-a9, cortex-a15, krait, denver. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.") diff --git a/libc/arch-arm/cortex-a53/cortex-a53.mk b/libc/arch-arm/cortex-a53/cortex-a53.mk index 9af03d94b..3ed80f283 100644 --- a/libc/arch-arm/cortex-a53/cortex-a53.mk +++ b/libc/arch-arm/cortex-a53/cortex-a53.mk @@ -1 +1,10 @@ -include bionic/libc/arch-arm/cortex-a15/cortex-a15.mk +libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memcpy.S \ + arch-arm/cortex-a15/bionic/memset.S \ + arch-arm/cortex-a15/bionic/strcat.S \ + arch-arm/cortex-a15/bionic/strcmp.S \ + arch-arm/cortex-a15/bionic/strcpy.S \ + arch-arm/cortex-a15/bionic/strlen.S \ + arch-arm/cortex-a15/bionic/__strcat_chk.S \ + arch-arm/cortex-a15/bionic/__strcpy_chk.S \ + arch-arm/krait/bionic/memmove.S diff --git a/libc/arch-arm/krait/bionic/memmove.S b/libc/arch-arm/krait/bionic/memmove.S new file mode 100644 index 000000000..698cc52ba --- /dev/null +++ b/libc/arch-arm/krait/bionic/memmove.S @@ -0,0 +1,219 @@ +/*************************************************************************** + Copyright (c) 2009-2014 The Linux Foundation. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of The Linux Foundation nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + ***************************************************************************/ + +/*************************************************************************** + * Neon memmove: Attempts to do a memmove with Neon registers if possible, + * Inputs: + * dest: The destination buffer + * src: The source buffer + * n: The size of the buffer to transfer + * Outputs: + * + ***************************************************************************/ + +#include +#include +/* + * These can be overridden in: + * device///BoardConfig.mk + * by setting the following: + * TARGET_USE_KRAIT_BIONIC_OPTIMIZATION := true + * TARGET_USE_KRAIT_PLD_SET := true + * TARGET_KRAIT_BIONIC_PLDOFFS := + * TARGET_KRAIT_BIONIC_PLDSIZE := + * TARGET_KRAIT_BIONIC_PLDTHRESH := + */ +#ifndef PLDOFFS +#define PLDOFFS (10) +#endif +#ifndef PLDTHRESH +#define PLDTHRESH (PLDOFFS) +#endif +#if (PLDOFFS < 5) +#error Routine does not support offsets less than 5 +#endif +#if (PLDTHRESH < PLDOFFS) +#error PLD threshold must be greater than or equal to the PLD offset +#endif +#ifndef PLDSIZE +#define PLDSIZE (64) +#endif + + .text + .syntax unified + .fpu neon + .thumb + .thumb_func + +ENTRY(bcopy) + //.cfi_startproc + mov r12, r0 + mov r0, r1 + mov r1, r12 + // Fall through to memmove + //.cfi_endproc +END(bcopy) + +ENTRY(memmove) +_memmove_words: + //.cfi_startproc + .save {r0, lr} + cmp r2, #0 + it ne + subsne r12, r0, r1 // Warning: do not combine these "it" blocks + it eq + bxeq lr +// memmove only if r1 < r0 < r1+r2 + cmp r0, r1 + itt ge + addge r12, r1, r2 + cmpge r12, r0 + it le + ble memcpy + cmp r2, #4 + it le + ble .Lneon_b2f_smallcopy_loop + push {r0, lr} + add r0, r0, r2 + add r1, r1, r2 + cmp r2, #64 + it ge + bge .Lneon_b2f_copy_64 + cmp r2, #32 + it ge + bge .Lneon_b2f_copy_32 + cmp r2, #8 + it ge + bge .Lneon_b2f_copy_8 + b .Lneon_b2f_copy_1 +.Lneon_b2f_copy_64: + mov r12, r2, lsr #6 + add r0, r0, #32 + add r1, r1, #32 + cmp r12, #PLDTHRESH + it le + ble .Lneon_b2f_copy_64_loop_nopld + sub r12, #PLDOFFS + sub lr, r1, #(PLDOFFS)*PLDSIZE +.Lneon_b2f_copy_64_loop_outer: + pld [lr] + sub r1, r1, #96 + sub r0, r0, #96 + vld1.32 {q0, q1}, [r1]! + vld1.32 {q2, q3}, [r1] + sub lr, lr, #64 + subs r12, r12, #1 + vst1.32 {q0, q1}, [r0]! + vst1.32 {q2, q3}, [r0] + it ne + bne .Lneon_b2f_copy_64_loop_outer + mov r12, #PLDOFFS +.Lneon_b2f_copy_64_loop_nopld: + sub r1, r1, #96 + sub r0, r0, #96 + vld1.32 {q8, q9}, [r1]! + vld1.32 {q10, q11}, [r1] + subs r12, r12, #1 + vst1.32 {q8, q9}, [r0]! + vst1.32 {q10, q11}, [r0] + it ne + bne .Lneon_b2f_copy_64_loop_nopld + ands r2, r2, #0x3f + it eq + beq .Lneon_memmove_done + sub r1, r1, #32 + sub r0, r0, #32 + cmp r2, #32 + it lt + blt .Lneon_b2f_copy_8 +.Lneon_b2f_copy_32: + sub r1, r1, #32 + sub r0, r0, #32 + vld1.32 {q0, q1}, [r1] + vst1.32 {q0, q1}, [r0] + ands r2, r2, #0x1f + it eq + beq .Lneon_memmove_done +.Lneon_b2f_copy_8: + movs r12, r2, lsr #0x3 + it eq + beq .Lneon_b2f_copy_1 +.Lneon_b2f_copy_8_loop: + sub r1, r1, #8 + sub r0, r0, #8 + vld1.32 {d0}, [r1] + subs r12, r12, #1 + vst1.32 {d0}, [r0] + it ne + bne .Lneon_b2f_copy_8_loop + ands r2, r2, #0x7 + beq .Lneon_memmove_done +.Lneon_b2f_copy_1: + movs r12, r2, lsl #29 + itttt mi + submi r1, r1, #4 + submi r0, r0, #4 + ldrmi r3, [r1] + strmi r3, [r0] + movs r2, r2, lsl #31 + itttt cs + subcs r1, r1, #2 + subcs r0, r0, #2 + ldrhcs r3, [r1] + strhcs r3, [r0] + itttt mi + submi r1, r1, #1 + submi r0, r0, #1 + ldrbmi r12, [r1] + strbmi r12, [r0] +.Lneon_memmove_done: + pop {r0, pc} +.Lneon_b2f_smallcopy_loop: + // 4 bytes or less + add r1, r1, r2 + add r0, r0, r2 + movs r12, r2, lsl #29 + itttt mi + submi r1, r1, #4 + submi r0, r0, #4 + ldrmi r3, [r1] + strmi r3, [r0] + movs r2, r2, lsl #31 + itttt cs + subcs r1, r1, #2 + subcs r0, r0, #2 + ldrhcs r3, [r1] + strhcs r3, [r0] + itttt mi + submi r1, r1, #1 + submi r0, r0, #1 + ldrbmi r12, [r1] + strbmi r12, [r0] + bx lr +// .cfi_endproc +END(memmove) + diff --git a/libc/arch-arm/krait/krait.mk b/libc/arch-arm/krait/krait.mk index 631ab6879..d82c89679 100644 --- a/libc/arch-arm/krait/krait.mk +++ b/libc/arch-arm/krait/krait.mk @@ -4,10 +4,11 @@ libc_bionic_src_files_arm += \ arch-arm/krait/bionic/strcmp.S \ arch-arm/krait/bionic/__strcat_chk.S \ arch-arm/krait/bionic/__strcpy_chk.S \ + arch-arm/krait/bionic/memmove.S + # Use cortex-a15 versions of strcat/strcpy/strlen and standard memmove libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/strcat.S \ arch-arm/cortex-a15/bionic/strcpy.S \ arch-arm/cortex-a15/bionic/strlen.S \ - bionic/memmove.c \ From c6684cdb21956d4455e2e74a76dfcf6f8d1659cc Mon Sep 17 00:00:00 2001 From: Lucas Crowthers Date: Fri, 15 Jun 2012 19:27:46 -0400 Subject: [PATCH 11/89] Bionic/libm: math optimizations A couple math function optimizations. It is a squashed version of previous commits. Change-Id: I09db8e265a121ebb4a14a3b52bef11664d6cc341 --- libm/Android.mk | 37 +- libm/arm/e_pow.S | 459 ++++++++++++++++++ libm/arm/e_sqrt.S | 36 ++ libm/arm/e_sqrtf.S | 36 ++ libm/arm/s_cos.S | 421 ++++++++++++++++ libm/arm/s_sin.S | 416 ++++++++++++++++ libm/upstream-freebsd/lib/msun/src/e_pow.c | 35 +- libm/upstream-freebsd/lib/msun/src/k_cos.c | 12 + libm/upstream-freebsd/lib/msun/src/k_sin.c | 11 + .../lib/msun/src/math_private.h | 12 + 10 files changed, 1465 insertions(+), 10 deletions(-) create mode 100644 libm/arm/e_pow.S create mode 100644 libm/arm/e_sqrt.S create mode 100644 libm/arm/e_sqrtf.S create mode 100644 libm/arm/s_cos.S create mode 100644 libm/arm/s_sin.S diff --git a/libm/Android.mk b/libm/Android.mk index 994caa0c6..77ec5d249 100644 --- a/libm/Android.mk +++ b/libm/Android.mk @@ -64,8 +64,6 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/e_scalbf.c \ upstream-freebsd/lib/msun/src/e_sinh.c \ upstream-freebsd/lib/msun/src/e_sinhf.c \ - upstream-freebsd/lib/msun/src/e_sqrt.c \ - upstream-freebsd/lib/msun/src/e_sqrtf.c \ upstream-freebsd/lib/msun/src/imprecise.c \ upstream-freebsd/lib/msun/src/k_cos.c \ upstream-freebsd/lib/msun/src/k_cosf.c \ @@ -96,7 +94,6 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_conjf.c \ upstream-freebsd/lib/msun/src/s_copysign.c \ upstream-freebsd/lib/msun/src/s_copysignf.c \ - upstream-freebsd/lib/msun/src/s_cos.c \ upstream-freebsd/lib/msun/src/s_cosf.c \ upstream-freebsd/lib/msun/src/s_cproj.c \ upstream-freebsd/lib/msun/src/s_cprojf.c \ @@ -161,7 +158,6 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_signgam.c \ upstream-freebsd/lib/msun/src/s_significand.c \ upstream-freebsd/lib/msun/src/s_significandf.c \ - upstream-freebsd/lib/msun/src/s_sin.c \ upstream-freebsd/lib/msun/src/s_sinf.c \ upstream-freebsd/lib/msun/src/s_tan.c \ upstream-freebsd/lib/msun/src/s_tanf.c \ @@ -173,7 +169,7 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/w_cabs.c \ upstream-freebsd/lib/msun/src/w_cabsf.c \ upstream-freebsd/lib/msun/src/w_drem.c \ - upstream-freebsd/lib/msun/src/w_dremf.c \ + upstream-freebsd/lib/msun/src/w_dremf.c libm_common_src_files += \ fake_long_double.c \ @@ -246,11 +242,33 @@ libm_common_cflags := \ # self recursions for lrint, lrintf, and lrintl. # BUG: 14225968 libm_common_cflags += -fno-builtin-rint -fno-builtin-rintf -fno-builtin-rintl - libm_common_includes := $(LOCAL_PATH)/upstream-freebsd/lib/msun/src/ libm_ld_includes := $(LOCAL_PATH)/upstream-freebsd/lib/msun/ld128/ +ifeq ($(TARGET_USE_QCOM_BIONIC_OPTIMIZATION),true) + libm_arm_src_files += \ + arm/e_pow.S \ + arm/s_cos.S \ + arm/s_sin.S \ + arm/e_sqrtf.S \ + arm/e_sqrt.S + libm_arm_cflags += -DQCOM_NEON_OPTIMIZATION -fno-if-conversion + libm_arm_includes += $(LOCAL_PATH)/../libc/ + + libm_arm64_src_files += \ + upstream-freebsd/lib/msun/src/s_cos.c \ + upstream-freebsd/lib/msun/src/s_sin.c \ + upstream-freebsd/lib/msun/src/e_sqrtf.c \ + upstream-freebsd/lib/msun/src/e_sqrt.c +else + libm_common_src_files += \ + upstream-freebsd/lib/msun/src/s_cos.c \ + upstream-freebsd/lib/msun/src/s_sin.c \ + upstream-freebsd/lib/msun/src/e_sqrtf.c \ + upstream-freebsd/lib/msun/src/e_sqrt.c +endif + # # libm.a for target. # @@ -264,11 +282,12 @@ LOCAL_SRC_FILES := $(libm_common_src_files) LOCAL_SYSTEM_SHARED_LIBRARIES := libc # arch-specific settings -LOCAL_C_INCLUDES_arm := $(LOCAL_PATH)/arm -LOCAL_SRC_FILES_arm := arm/fenv.c +LOCAL_CFLAGS_arm := $(libm_arm_cflags) +LOCAL_C_INCLUDES_arm := $(LOCAL_PATH)/arm $(libm_arm_includes) +LOCAL_SRC_FILES_arm := arm/fenv.c $(libm_arm_src_files) LOCAL_C_INCLUDES_arm64 := $(libm_ld_includes) -LOCAL_SRC_FILES_arm64 := arm64/fenv.c $(libm_ld_src_files) +LOCAL_SRC_FILES_arm64 := arm64/fenv.c $(libm_ld_src_files) $(libm_arm64_src_files) LOCAL_C_INCLUDES_x86 := $(LOCAL_PATH)/i387 LOCAL_SRC_FILES_x86 := i387/fenv.c diff --git a/libm/arm/e_pow.S b/libm/arm/e_pow.S new file mode 100644 index 000000000..a730fa657 --- /dev/null +++ b/libm/arm/e_pow.S @@ -0,0 +1,459 @@ +@ Copyright (c) 2009-2013 The Linux Foundation. All rights reserved. +@ +@ Redistribution and use in source and binary forms, with or without +@ modification, are permitted provided that the following conditions are met: +@ * Redistributions of source code must retain the above copyright +@ notice, this list of conditions and the following disclaimer. +@ * Redistributions in binary form must reproduce the above copyright +@ notice, this list of conditions and the following disclaimer in the +@ documentation and/or other materials provided with the distribution. +@ * Neither the name of The Linux Foundation nor the names of its contributors may +@ be used to endorse or promote products derived from this software +@ without specific prior written permission. +@ +@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +@ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +@ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +@ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +@ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +@ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +@ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +@ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +@ POSSIBILITY OF SUCH DAMAGE. + + +#include +#include + +@ Values which exist the program lifetime: +#define HIGH_WORD_MASK d31 +#define EXPONENT_MASK d30 +#define int_1 d29 +#define double_1 d28 +@ sign and 2^int_n fixup: +#define maxrange r12 +#define expadjustment d7 +#define literals r10 +@ Values which exist within both polynomial implementations: +#define int_n d2 +#define int_n_low s4 +#define int_n_high s5 +#define double_n d3 +#define k1 d27 +#define k2 d26 +#define k3 d25 +#define k4 d24 +@ Values which cross the boundaries between polynomial implementations: +#define ss d16 +#define ss2 d17 +#define ss4 d18 +#define Result d0 +#define Return_hw r1 +#define Return_lw r0 +#define ylg2x d0 +@ Intermediate values only needed sometimes: +@ initial (sorted in approximate order of availability for overwriting): +#define x_hw r1 +#define x_lw r0 +#define y_hw r3 +#define y_lw r2 +#define x d0 +#define bp d4 +#define y d1 +@ log series: +#define u d19 +#define v d20 +#define lg2coeff d21 +#define bpa d5 +#define bpb d3 +#define lg2const d6 +#define xmantissa r8 +#define twoto1o5 r4 +#define twoto3o5 r5 +#define ix r6 +#define iEXP_MASK r7 +@ exp input setup: +#define twoto1o8mask d3 +#define twoto1o4mask d4 +#define twoto1o2mask d1 +#define ylg2x_round_offset d16 +#define ylg2x_temp d17 +#define yn_temp d18 +#define yn_round_offset d19 +#define ln2 d5 +@ Careful, overwriting HIGH_WORD_MASK, reset it if you need it again ... +#define rounded_exponent d31 +@ exp series: +#define k5 d23 +#define k6 d22 +#define k7 d21 +#define k8 d20 +#define ss3 d19 +@ overwrite double_1 (we're done with it by now) +#define k0 d28 +#define twoto1o4 d6 + +@instructions that gas doesn't like to encode correctly: +#define vmov_f64 fconstd +#define vmov_f32 fconsts +#define vmovne_f64 fconstdne + +#define KRAIT_NO_AAPCS_VFP_MODE + +ENTRY(pow) +#if defined(KRAIT_NO_AAPCS_VFP_MODE) + @ ARM ABI has inputs coming in via r registers, lets move to a d register + vmov x, x_lw, x_hw +#endif + push {r4, r5, r6, r7, r8, r9, r10, lr} + + movw maxrange, #0x0000 + movt maxrange, #0x4010 + + @ pre-staged bp values + vldr bpa, .LbpA + vldr bpb, .LbpB + @ load two fifths into constant term in case we need it due to offsets + vldr lg2const, .Ltwofifths + + @ bp is initially 1.0, may adjust later based on x value + vmov_f64 bp, #0x70 + + @ extract the mantissa from x for scaled value comparisons + lsl xmantissa, x_hw, #12 + + @ twoto1o5 = 2^(1/5) (input bracketing) + movw twoto1o5, #0x186c + movt twoto1o5, #0x2611 + @ twoto3o5 = 2^(3/5) (input bracketing) + movw twoto3o5, #0x003b + movt twoto3o5, #0x8406 + + @ finish extracting xmantissa + orr xmantissa, xmantissa, x_lw, lsr #20 + + @ begin preparing a mask for normalization + vmov.i64 HIGH_WORD_MASK, #0xffffffff00000000 + + @ double_1 = (double) 1.0 + vmov_f64 double_1, #0x70 + +#if defined(KRAIT_NO_AAPCS_VFP_MODE) + @ move y from r registers to a d register + vmov y, y_lw, y_hw +#endif + + cmp xmantissa, twoto1o5 + + vshl.i64 EXPONENT_MASK, HIGH_WORD_MASK, #20 + vshr.u64 int_1, HIGH_WORD_MASK, #63 + + adr literals, .LliteralTable + + bhi .Lxgt2to1over5 + @ zero out lg2 constant term if don't offset our input + vsub.f64 lg2const, lg2const, lg2const + b .Lxle2to1over5 + +.Lxgt2to1over5: + @ if normalized x > 2^(1/5), bp = 1 + (2^(2/5)-1) = 2^(2/5) + vadd.f64 bp, bp, bpa + +.Lxle2to1over5: + @ will need ln2 for various things + vldr ln2, .Lln2 + + cmp xmantissa, twoto3o5 +@@@@ X Value Normalization @@@@ + + @ ss = abs(x) 2^(-1024) + vbic.i64 ss, x, EXPONENT_MASK + + @ N = (floor(log2(x)) + 0x3ff) * 2^52 + vand.i64 int_n, x, EXPONENT_MASK + + bls .Lxle2to3over5 + @ if normalized x > 2^(3/5), bp = 2^(2/5) + (2^(4/5) - 2^(2/5) = 2^(4/5) + vadd.f64 bp, bp, bpb + vadd.f64 lg2const, lg2const, lg2const + +.Lxle2to3over5: + + cmp x_hw, maxrange + cmpls y_hw, maxrange + movt maxrange, #0x3f00 + cmpls maxrange, x_hw + + @ load log2 polynomial series constants + vldm literals!, {k4, k3, k2, k1} + + @ s = abs(x) 2^(-floor(log2(x))) (normalize abs(x) to around 1) + vorr.i64 ss, ss, double_1 + +@@@@ 3/2 (Log(bp(1+s)/(1-s))) input computation (s = (x-bp)/(x+bp)) @@@@ + + vsub.f64 u, ss, bp + vadd.f64 v, ss, bp + + bhi .LuseFullImpl + + @ s = (x-1)/(x+1) + vdiv.f64 ss, u, v + + @ load 2/(3log2) into lg2coeff + vldr lg2coeff, .Ltwooverthreeln2 + + @ N = floor(log2(x)) * 2^52 + vsub.i64 int_n, int_n, double_1 + +@@@@ 3/2 (Log(bp(1+s)/(1-s))) polynomial series @@@@ + + @ ss2 = ((x-dp)/(x+dp))^2 + vmul.f64 ss2, ss, ss + @ ylg2x = 3.0 + vmov_f64 ylg2x, #8 + vmul.f64 ss4, ss2, ss2 + + @ todo: useful later for two-way clamp + vmul.f64 lg2coeff, lg2coeff, y + + @ N = floor(log2(x)) + vshr.s64 int_n, int_n, #52 + + @ k3 = ss^2 * L4 + L3 + vmla.f64 k3, ss2, k4 + + @ k1 = ss^2 * L2 + L1 + vmla.f64 k1, ss2, k2 + + @ scale ss by 2/(3 ln 2) + vmul.f64 lg2coeff, ss, lg2coeff + + @ ylg2x = 3.0 + s^2 + vadd.f64 ylg2x, ylg2x, ss2 + + vcvt.f64.s32 double_n, int_n_low + + @ k1 = s^4 (s^2 L4 + L3) + s^2 L2 + L1 + vmla.f64 k1, ss4, k3 + + @ add in constant term + vadd.f64 double_n, lg2const + + @ ylg2x = 3.0 + s^2 + s^4 (s^4 (s^2 L4 + L3) + s^2 L2 + L1) + vmla.f64 ylg2x, ss4, k1 + + @ ylg2x = y 2 s / (3 ln(2)) (3.0 + s^2 + s^4 (s^4(s^2 L4 + L3) + s^2 L2 + L1) + vmul.f64 ylg2x, lg2coeff, ylg2x + +@@@@ Compute input to Exp(s) (s = y(n + log2(x)) - (floor(8 yn + 1)/8 + floor(8 ylog2(x) + 1)/8) @@@@@ + + @ mask to extract bit 1 (2^-2 from our fixed-point representation) + vshl.u64 twoto1o4mask, int_1, #1 + + @ double_n = y * n + vmul.f64 double_n, double_n, y + + @ Load 2^(1/4) for later computations + vldr twoto1o4, .Ltwoto1o4 + + @ either add or subtract one based on the sign of double_n and ylg2x + vshr.s64 ylg2x_round_offset, ylg2x, #62 + vshr.s64 yn_round_offset, double_n, #62 + + @ move unmodified y*lg2x into temp space + vmov ylg2x_temp, ylg2x + @ compute floor(8 y * n + 1)/8 + @ and floor(8 y (log2(x)) + 1)/8 + vcvt.s32.f64 ylg2x, ylg2x, #3 + @ move unmodified y*n into temp space + vmov yn_temp, double_n + vcvt.s32.f64 double_n, double_n, #3 + + @ load exp polynomial series constants + vldm literals!, {k8, k7, k6, k5, k4, k3, k2, k1} + + @ mask to extract bit 2 (2^-1 from our fixed-point representation) + vshl.u64 twoto1o2mask, int_1, #2 + + @ make rounding offsets either 1 or -1 instead of 0 or -2 + vorr.u64 ylg2x_round_offset, ylg2x_round_offset, int_1 + vorr.u64 yn_round_offset, yn_round_offset, int_1 + + @ round up to the nearest 1/8th + vadd.s32 ylg2x, ylg2x, ylg2x_round_offset + vadd.s32 double_n, double_n, yn_round_offset + + @ clear out round-up bit for y log2(x) + vbic.s32 ylg2x, ylg2x, int_1 + @ clear out round-up bit for yn + vbic.s32 double_n, double_n, int_1 + @ add together the (fixed precision) rounded parts + vadd.s64 rounded_exponent, double_n, ylg2x + @ turn int_n into a double with value 2^int_n + vshl.i64 int_n, rounded_exponent, #49 + @ compute masks for 2^(1/4) and 2^(1/2) fixups for fractional part of fixed-precision rounded values: + vand.u64 twoto1o4mask, twoto1o4mask, rounded_exponent + vand.u64 twoto1o2mask, twoto1o2mask, rounded_exponent + + @ convert back into floating point, double_n now holds (double) floor(8 y * n + 1)/8 + @ ylg2x now holds (double) floor(8 y * log2(x) + 1)/8 + vcvt.f64.s32 ylg2x, ylg2x, #3 + vcvt.f64.s32 double_n, double_n, #3 + + @ put the 2 bit (0.5) through the roof of twoto1o2mask (make it 0x0 or 0xffffffffffffffff) + vqshl.u64 twoto1o2mask, twoto1o2mask, #62 + @ put the 1 bit (0.25) through the roof of twoto1o4mask (make it 0x0 or 0xffffffffffffffff) + vqshl.u64 twoto1o4mask, twoto1o4mask, #63 + + @ center y*log2(x) fractional part between -0.125 and 0.125 by subtracting (double) floor(8 y * log2(x) + 1)/8 + vsub.f64 ylg2x_temp, ylg2x_temp, ylg2x + @ center y*n fractional part between -0.125 and 0.125 by subtracting (double) floor(8 y * n + 1)/8 + vsub.f64 yn_temp, yn_temp, double_n + + @ Add fractional parts of yn and y log2(x) together + vadd.f64 ss, ylg2x_temp, yn_temp + + @ Result = 1.0 (offset for exp(s) series) + vmov_f64 Result, #0x70 + + @ multiply fractional part of y * log2(x) by ln(2) + vmul.f64 ss, ln2, ss + +@@@@ 10th order polynomial series for Exp(s) @@@@ + + @ ss2 = (ss)^2 + vmul.f64 ss2, ss, ss + + @ twoto1o2mask = twoto1o2mask & twoto1o4 + vand.u64 twoto1o2mask, twoto1o2mask, twoto1o4 + @ twoto1o2mask = twoto1o2mask & twoto1o4 + vand.u64 twoto1o4mask, twoto1o4mask, twoto1o4 + + @ Result = 1.0 + ss + vadd.f64 Result, Result, ss + + @ k7 = ss k8 + k7 + vmla.f64 k7, ss, k8 + + @ ss4 = (ss*ss) * (ss*ss) + vmul.f64 ss4, ss2, ss2 + + @ twoto1o2mask = twoto1o2mask | (double) 1.0 - results in either 1.0 or 2^(1/4) in twoto1o2mask + vorr.u64 twoto1o2mask, twoto1o2mask, double_1 + @ twoto1o2mask = twoto1o4mask | (double) 1.0 - results in either 1.0 or 2^(1/4) in twoto1o4mask + vorr.u64 twoto1o4mask, twoto1o4mask, double_1 + + @ TODO: should setup sign here, expadjustment = 1.0 + vmov_f64 expadjustment, #0x70 + + @ ss3 = (ss*ss) * ss + vmul.f64 ss3, ss2, ss + + @ k0 = 1/2 (first non-unity coefficient) + vmov_f64 k0, #0x60 + + @ Mask out non-exponent bits to make sure we have just 2^int_n + vand.i64 int_n, int_n, EXPONENT_MASK + + @ square twoto1o2mask to get 1.0 or 2^(1/2) + vmul.f64 twoto1o2mask, twoto1o2mask, twoto1o2mask + @ multiply twoto2o4mask into the exponent output adjustment value + vmul.f64 expadjustment, expadjustment, twoto1o4mask + + @ k5 = ss k6 + k5 + vmla.f64 k5, ss, k6 + + @ k3 = ss k4 + k3 + vmla.f64 k3, ss, k4 + + @ k1 = ss k2 + k1 + vmla.f64 k1, ss, k2 + + @ multiply twoto1o2mask into exponent output adjustment value + vmul.f64 expadjustment, expadjustment, twoto1o2mask + + @ k5 = ss^2 ( ss k8 + k7 ) + ss k6 + k5 + vmla.f64 k5, ss2, k7 + + @ k1 = ss^2 ( ss k4 + k3 ) + ss k2 + k1 + vmla.f64 k1, ss2, k3 + + @ Result = 1.0 + ss + 1/2 ss^2 + vmla.f64 Result, ss2, k0 + + @ Adjust int_n so that it's a double precision value that can be multiplied by Result + vadd.i64 expadjustment, int_n, expadjustment + + @ k1 = ss^4 ( ss^2 ( ss k8 + k7 ) + ss k6 + k5 ) + ss^2 ( ss k4 + k3 ) + ss k2 + k1 + vmla.f64 k1, ss4, k5 + + @ Result = 1.0 + ss + 1/2 ss^2 + ss^3 ( ss^4 ( ss^2 ( ss k8 + k7 ) + ss k6 + k5 ) + ss^2 ( ss k4 + k3 ) + ss k2 + k1 ) + vmla.f64 Result, ss3, k1 + + @ multiply by adjustment (sign*(rounding ? sqrt(2) : 1) * 2^int_n) + vmul.f64 Result, expadjustment, Result + +.LleavePow: +#if defined(KRAIT_NO_AAPCS_VFP_MODE) + @ return Result (FP) + vmov Return_lw, Return_hw, Result +#endif +.LleavePowDirect: + @ leave directly returning whatever is in Return_lw and Return_hw + pop {r4, r5, r6, r7, r8, r9, r10, pc} + +.LuseFullImpl: + pop {r4, r5, r6, r7, r8, r9, r10, lr} + b __full_ieee754_pow + +.align 6 +.LliteralTable: +@ Least-sqares tuned constants for 11th order (log2((1+s)/(1-s)): +.LL4: @ ~3/11 + .long 0x53a79915, 0x3fd1b108 +.LL3: @ ~1/3 + .long 0x9ca0567a, 0x3fd554fa +.LL2: @ ~3/7 + .long 0x1408e660, 0x3fdb6db7 +.LL1: @ ~3/5 + .long 0x332D4313, 0x3fe33333 + +@ Least-squares tuned constants for 10th order exp(s): +.LE10: @ ~1/3628800 + .long 0x25c7ba0a, 0x3e92819b +.LE9: @ ~1/362880 + .long 0x9499b49c, 0x3ec72294 +.LE8: @ ~1/40320 + .long 0xabb79d95, 0x3efa019f +.LE7: @ ~1/5040 + .long 0x8723aeaa, 0x3f2a019f +.LE6: @ ~1/720 + .long 0x16c76a94, 0x3f56c16c +.LE5: @ ~1/120 + .long 0x11185da8, 0x3f811111 +.LE4: @ ~1/24 + .long 0x5555551c, 0x3fa55555 +.LE3: @ ~1/6 + .long 0x555554db, 0x3fc55555 + +.LbpA: @ (2^(2/5) - 1) + .long 0x4ee54db1, 0x3fd472d1 + +.LbpB: @ (2^(4/5) - 2^(2/5)) + .long 0x1c8a36cf, 0x3fdafb62 + +.Ltwofifths: @ + .long 0x9999999a, 0x3fd99999 + +.Ltwooverthreeln2: + .long 0xDC3A03FD, 0x3FEEC709 + +.Lln2: @ ln(2) + .long 0xFEFA39EF, 0x3FE62E42 + +.Ltwoto1o4: @ 2^1/4 + .long 0x0a31b715, 0x3ff306fe +END(pow) diff --git a/libm/arm/e_sqrt.S b/libm/arm/e_sqrt.S new file mode 100644 index 000000000..f2c88ff98 --- /dev/null +++ b/libm/arm/e_sqrt.S @@ -0,0 +1,36 @@ +@ Copyright (c) 2013, The Linux Foundation. All rights reserved. +@ +@ Redistribution and use in source and binary forms, with or without +@ modification, are permitted provided that the following conditions are +@ met: +@ * Redistributions of source code must retain the above copyright +@ notice, this list of conditions and the following disclaimer. +@ * Redistributions in binary form must reproduce the above +@ copyright notice, this list of conditions and the following +@ disclaimer in the documentation and/or other materials provided +@ with the distribution. +@ * Neither the name of The Linux Foundation nor the names of its +@ contributors may be used to endorse or promote products derived +@ from this software without specific prior written permission. +@ +@ THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +@ WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +@ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +@ BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +@ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +@ BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +@ OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +@ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include + +ENTRY(sqrt) + vmov.f64 d0, r0, r1 + vsqrt.f64 d0, d0 + vmov.f64 r0, r1, d0 + bx lr +END(sqrt) diff --git a/libm/arm/e_sqrtf.S b/libm/arm/e_sqrtf.S new file mode 100644 index 000000000..b14a77b3e --- /dev/null +++ b/libm/arm/e_sqrtf.S @@ -0,0 +1,36 @@ +@ Copyright (c) 2013, The Linux Foundation. All rights reserved. +@ +@ Redistribution and use in source and binary forms, with or without +@ modification, are permitted provided that the following conditions are +@ met: +@ * Redistributions of source code must retain the above copyright +@ notice, this list of conditions and the following disclaimer. +@ * Redistributions in binary form must reproduce the above +@ copyright notice, this list of conditions and the following +@ disclaimer in the documentation and/or other materials provided +@ with the distribution. +@ * Neither the name of The Linux Foundation nor the names of its +@ contributors may be used to endorse or promote products derived +@ from this software without specific prior written permission. +@ +@ THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +@ WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +@ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +@ BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +@ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +@ BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +@ OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +@ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include + +ENTRY(sqrtf) + vmov.f32 s0, r0 + vsqrt.f32 s0, s0 + vmov.f32 r0, s0 + bx lr +END(sqrtf) diff --git a/libm/arm/s_cos.S b/libm/arm/s_cos.S new file mode 100644 index 000000000..eae161e5a --- /dev/null +++ b/libm/arm/s_cos.S @@ -0,0 +1,421 @@ +@ Copyright (c) 2012, The Linux Foundation. All rights reserved. +@ +@ Redistribution and use in source and binary forms, with or without +@ modification, are permitted provided that the following conditions are +@ met: +@ * Redistributions of source code must retain the above copyright +@ notice, this list of conditions and the following disclaimer. +@ * Redistributions in binary form must reproduce the above +@ copyright notice, this list of conditions and the following +@ disclaimer in the documentation and/or other materials provided +@ with the distribution. +@ * Neither the name of The Linux Foundation nor the names of its +@ contributors may be used to endorse or promote products derived +@ from this software without specific prior written permission. +@ +@ THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +@ WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +@ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +@ BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +@ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +@ BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +@ OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +@ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +@ +@ Additional notices preserved for attributions purposes only. +@ +@ ==================================================== +@ Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +@ +@ Developed at SunSoft, a Sun Microsystems, Inc. business. +@ Permission to use, copy, modify, and distribute this +@ software is freely granted, provided that this notice +@ is preserved. +@ ==================================================== +@ +@ ==================================================== +@ Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +@ +@ Developed at SunPro, a Sun Microsystems, Inc. business. +@ Permission to use, copy, modify, and distribute this +@ software is freely granted, provided that this notice +@ is preserved. +@ ==================================================== + +#include +#include + +#define vmov_f64 fconstd + +ENTRY(cosl) +END(cosl) +ENTRY(cos) + push {r4, r6, r7, lr} + vmov d0, r0, r1 + mov r2, r0 + mov r3, r1 + movw r1, #0x21fb + movt r1, #0x3fe9 + mov r4, r3 + bic r3, r3, #0x80000000 + sub sp, sp, #48 + cmp r3, r1 + bgt .Lxgtpio4 + cmp r3, #0x3e400000 + bge .Lxnottiny + vcvt.s32.f64 s15, d0 + vmov r3, s15 + cmp r3, #0 + beq .Lreturnone +.Lxnottiny: + vmov.i64 d1, #0 + bl __kernel_cos +.Lleave_cos: + vmov r0, r1, d0 +.Lleave_cos_direct: + add sp, sp, #48 + pop {r4, r6, r7, pc} +.Lxgtpio4: + movw r2, #0xffff + movt r2, #0x7fef + cmp r3, r2 + bgt .LxisNaN + movw r0, #0xd97b + movt r0, #0x4002 + cmp r3, r0 + movw r2, #0x21fb + bgt .Lxge3pio4 + cmp r4, #0 + movt r2, #0x3ff9 + ble .Lsmallxisnegative + vldr d16, .Lpio2_1 + cmp r3, r2 + vsub.f64 d16, d0, d16 + beq .Lxnearpio2 + vldr d17, .Lpio2_1t +.Lfinalizesmallxremainder: + vsub.f64 d0, d16, d17 + vsub.f64 d16, d16, d0 + vstr d0, [sp, #8] + vsub.f64 d1, d16, d17 + vstr d1, [sp, #16] +.Lnmod3is1: + mov r0, #1 + bl __kernel_sin + vneg.f64 d0, d0 + b .Lleave_cos +.Lreturnone: + mov r0, #0 + movw r1, #0x0000 + movt r1, #0x3ff0 + vmov_f64 d0, #0x70 + b .Lleave_cos_direct +.LxisNaN: + vsub.f64 d0, d0, d0 + b .Lleave_cos +.Lxge3pio4: + movt r2, #0x4139 + cmp r3, r2 + bgt .Lxgigantic + vmov_f64 d3, #0x60 + vldr d2, .Linvpio2 + vldr d18, .Lpio2_1 + vabs.f64 d16, d0 + vmla.f64 d3, d16, d2 + vcvt.s32.f64 s3, d3 + vcvt.f64.s32 d17, s3 + vmov r0, s3 + cmp r0, #31 + vmls.f64 d16, d17, d18 + vldr d18, .Lpio2_1t + vmul.f64 d18, d17, d18 + bgt .Lcomputeremainder + ldr r2, .Lnpio2_hw_ptr + sub lr, r0, #1 +.LPICnpio2_hw0: + add r12, pc, r2 + ldr r1, [r12, lr, lsl #2] + cmp r3, r1 + beq .Lcomputeremainder +.Lfinishthirditeration: + vsub.f64 d0, d16, d18 + vstr d0, [sp, #8] +.Lfinishcomputingremainder: + vsub.f64 d16, d16, d0 + cmp r4, #0 + vsub.f64 d1, d16, d18 + vstr d1, [sp, #16] + blt .Lhandlenegativex +.Lselectregion: + and r0, r0, #3 + cmp r0, #1 + beq .Lnmod3is1 + cmp r0, #2 + beq .Lnmod3is2 + cmp r0, #0 + bne .Lnmod3is0 + bl __kernel_cos + b .Lleave_cos +.Lxgigantic: + asr r2, r3, #20 + vmov r6, r7, d0 + sub r2, r2, #1040 + mov r0, r6 + sub r2, r2, #6 + vldr d16, .Ltwo24 + sub r1, r3, r2, lsl #20 + vmov d18, r0, r1 + vcvt.s32.f64 s15, d18 + add r1, sp, #48 + mov r3, #3 + vcvt.f64.s32 d17, s15 + vsub.f64 d18, d18, d17 + vstr d17, [sp, #24] + vmul.f64 d18, d18, d16 + vcvt.s32.f64 s15, d18 + vcvt.f64.s32 d17, s15 + vsub.f64 d18, d18, d17 + vstr d17, [sp, #32] + vmul.f64 d16, d18, d16 + fcmpzd d16 + vstmdb r1!, {d16} + vmrs APSR_nzcv, fpscr + bne .Lprocessnonzeroterm +.Lskipzeroterms: + vldmdb r1!, {d16} + sub r3, r3, #1 + fcmpzd d16 + vmrs APSR_nzcv, fpscr + beq .Lskipzeroterms +.Lprocessnonzeroterm: + ldr r12, .Ltwo_over_pi_ptr + add r0, sp, #24 + add r1, sp, #8 +.LPICtwo_over_pi0: + add lr, pc, r12 + mov r12, #2 + str lr, [sp, #4] + str r12, [sp] + bl __kernel_rem_pio2 + cmp r4, #0 + vldr d0, [sp, #8] + blt .Lhandlenegativxalso + vldr d1, [sp, #16] + b .Lselectregion +.Lxnearpio2: + vldr d17, .Lpio2_2 + vsub.f64 d16, d16, d17 + vldr d17, .Lpio2_2t + b .Lfinalizesmallxremainder +.Lsmallxisnegative: + vldr d1, .Lpio2_1 + cmp r3, r2 + vadd.f64 d16, d0, d1 + beq .Lxnearnegpio2 + vldr d17, .Lpio2_1t +.Lfinalizesmallnegxremainder: + vadd.f64 d0, d16, d17 + vsub.f64 d16, d16, d0 + vstr d0, [sp, #8] + vadd.f64 d1, d16, d17 + vstr d1, [sp, #16] +.Lnmod3is0: + mov r0, #1 + bl __kernel_sin + b .Lleave_cos +.Lnmod3is2: + bl __kernel_cos + vneg.f64 d0, d0 + b .Lleave_cos +.Lcomputeremainder: + vsub.f64 d0, d16, d18 + asr r1, r3, #20 + vmov r2, r3, d0 + ubfx r3, r3, #20, #11 + rsb r3, r3, r1 + vstr d0, [sp, #8] + cmp r3, #16 + ble .Lfinishcomputingremainder + vldr d18, .Lpio2_2 + vmul.f64 d20, d17, d18 + vsub.f64 d19, d16, d20 + vsub.f64 d16, d16, d19 + vsub.f64 d18, d16, d20 + vldr d16, .Lpio2_2t + vnmls.f64 d18, d17, d16 + vsub.f64 d0, d19, d18 + vmov r2, r3, d0 + ubfx r3, r3, #20, #11 + rsb r1, r3, r1 + vstr d0, [sp, #8] + cmp r1, #49 + ble .Lfinishseconditeration + vldr d5, .Lpio2_3 + vmul.f64 d20, d17, d5 + vsub.f64 d16, d19, d20 + vsub.f64 d4, d19, d16 + vldr d19, .Lpio2_3t + vsub.f64 d18, d4, d20 + vnmls.f64 d18, d17, d19 + b .Lfinishthirditeration +.Lhandlenegativex: + vneg.f64 d0, d0 + rsb r0, r0, #0 + vneg.f64 d1, d1 + vstr d0, [sp, #8] + vstr d1, [sp, #16] + b .Lselectregion +.Lfinishseconditeration: + vmov d16, d19 + b .Lfinishcomputingremainder +.Lxnearnegpio2: + vldr d0, .Lpio2_2 + vldr d17, .Lpio2_2t + vadd.f64 d16, d16, d0 + b .Lfinalizesmallnegxremainder +.Lhandlenegativxalso: + vldr d6, [sp, #16] + vneg.f64 d0, d0 + rsb r0, r0, #0 + vneg.f64 d1, d6 + vstr d0, [sp, #8] + vstr d1, [sp, #16] + b .Lselectregion + +.align 3 +.Lpio2_1: + .word 0x54400000, 0x3ff921fb +.Lpio2_1t: + .word 0x1a626331, 0x3dd0b461 +.Linvpio2: + .word 0x6dc9c883, 0x3fe45f30 +.Ltwo24: + .word 0x00000000, 0x41700000 +.Lpio2_2: + .word 0x1a600000, 0x3dd0b461 +.Lpio2_2t: + .word 0x2e037073, 0x3ba3198a +.Lpio2_3: + .word 0x2e000000, 0x3ba3198a +.Lpio2_3t: + .word 0x252049c1, 0x397b839a +.Lnpio2_hw_ptr: + .word .Lnpio2_hw-(.LPICnpio2_hw0+8) +.Ltwo_over_pi_ptr: + .word .Ltwo_over_pi-(.LPICtwo_over_pi0+8) +END(cos) + + .section .rodata.npio2_hw,"a",%progbits + .align 2 +.Lnpio2_hw = . + 0 + .type npio2_hw, %object + .size npio2_hw, 128 +npio2_hw: + .word 0x3ff921fb + .word 0x400921fb + .word 0x4012d97c + .word 0x401921fb + .word 0x401f6a7a + .word 0x4022d97c + .word 0x4025fdbb + .word 0x402921fb + .word 0x402c463a + .word 0x402f6a7a + .word 0x4031475c + .word 0x4032d97c + .word 0x40346b9c + .word 0x4035fdbb + .word 0x40378fdb + .word 0x403921fb + .word 0x403ab41b + .word 0x403c463a + .word 0x403dd85a + .word 0x403f6a7a + .word 0x40407e4c + .word 0x4041475c + .word 0x4042106c + .word 0x4042d97c + .word 0x4043a28c + .word 0x40446b9c + .word 0x404534ac + .word 0x4045fdbb + .word 0x4046c6cb + .word 0x40478fdb + .word 0x404858eb + .word 0x404921fb + + .section .rodata.two_over_pi,"a",%progbits + .align 2 +.Ltwo_over_pi = . + 0 + .type two_over_pi, %object + .size two_over_pi, 264 +two_over_pi: + .word 0x00a2f983 + .word 0x006e4e44 + .word 0x001529fc + .word 0x002757d1 + .word 0x00f534dd + .word 0x00c0db62 + .word 0x0095993c + .word 0x00439041 + .word 0x00fe5163 + .word 0x00abdebb + .word 0x00c561b7 + .word 0x00246e3a + .word 0x00424dd2 + .word 0x00e00649 + .word 0x002eea09 + .word 0x00d1921c + .word 0x00fe1deb + .word 0x001cb129 + .word 0x00a73ee8 + .word 0x008235f5 + .word 0x002ebb44 + .word 0x0084e99c + .word 0x007026b4 + .word 0x005f7e41 + .word 0x003991d6 + .word 0x00398353 + .word 0x0039f49c + .word 0x00845f8b + .word 0x00bdf928 + .word 0x003b1ff8 + .word 0x0097ffde + .word 0x0005980f + .word 0x00ef2f11 + .word 0x008b5a0a + .word 0x006d1f6d + .word 0x00367ecf + .word 0x0027cb09 + .word 0x00b74f46 + .word 0x003f669e + .word 0x005fea2d + .word 0x007527ba + .word 0x00c7ebe5 + .word 0x00f17b3d + .word 0x000739f7 + .word 0x008a5292 + .word 0x00ea6bfb + .word 0x005fb11f + .word 0x008d5d08 + .word 0x00560330 + .word 0x0046fc7b + .word 0x006babf0 + .word 0x00cfbc20 + .word 0x009af436 + .word 0x001da9e3 + .word 0x0091615e + .word 0x00e61b08 + .word 0x00659985 + .word 0x005f14a0 + .word 0x0068408d + .word 0x00ffd880 + .word 0x004d7327 + .word 0x00310606 + .word 0x001556ca + .word 0x0073a8c9 + .word 0x0060e27b + .word 0x00c08c6b diff --git a/libm/arm/s_sin.S b/libm/arm/s_sin.S new file mode 100644 index 000000000..3c449154b --- /dev/null +++ b/libm/arm/s_sin.S @@ -0,0 +1,416 @@ +@ Copyright (c) 2012, The Linux Foundation. All rights reserved. +@ +@ Redistribution and use in source and binary forms, with or without +@ modification, are permitted provided that the following conditions are +@ met: +@ * Redistributions of source code must retain the above copyright +@ notice, this list of conditions and the following disclaimer. +@ * Redistributions in binary form must reproduce the above +@ copyright notice, this list of conditions and the following +@ disclaimer in the documentation and/or other materials provided +@ with the distribution. +@ * Neither the name of The Linux Foundation nor the names of its +@ contributors may be used to endorse or promote products derived +@ from this software without specific prior written permission. +@ +@ THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED +@ WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT +@ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +@ BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +@ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +@ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +@ BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +@ OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +@ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +@ +@ Additional notices preserved for attributions purposes only. +@ +@ ==================================================== +@ Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +@ +@ Developed at SunSoft, a Sun Microsystems, Inc. business. +@ Permission to use, copy, modify, and distribute this +@ software is freely granted, provided that this notice +@ is preserved. +@ ==================================================== +@ +@ ==================================================== +@ Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +@ +@ Developed at SunPro, a Sun Microsystems, Inc. business. +@ Permission to use, copy, modify, and distribute this +@ software is freely granted, provided that this notice +@ is preserved. +@ ==================================================== + +#include +#include + +#define vmov_f64 fconstd + +ENTRY(sinl) +END(sinl) +ENTRY(sin) + push {r4, r6, r7, lr} + vmov d0, r0, r1 + mov r2, r0 + mov r3, r1 + movw r1, #0x21fb + movt r1, #0x3fe9 + mov r4, r3 + bic r3, r3, #0x80000000 + sub sp, sp, #48 + cmp r3, r1 + bgt .Lxgtpio4 + cmp r3, #0x3e400000 + bge .Lxnottiny + vcvt.s32.f64 s15, d0 + vmov r3, s15 + cmp r3, #0 + bne .Lxnottiny +.Lleave_sin: + vmov r0, r1, d0 + add sp, sp, #48 + pop {r4, r6, r7, pc} +.Lxgtpio4: + movw r2, #0xffff + movt r2, #0x7fef + cmp r3, r2 + bgt .LxisNaN + movw r0, #0xd97b + movt r0, #0x4002 + cmp r3, r0 + movw r2, #0x21fb + bgt .Lxge3pio4 + cmp r4, #0 + movt r2, #0x3ff9 + ble .Lsmallxisnegative + vldr d16, .Lpio2_1 + cmp r3, r2 + vsub.f64 d16, d0, d16 + beq .Lxnearpio2 + vldr d17, .Lpio2_1t +.Lfinalizesmallxremainder: + vsub.f64 d0, d16, d17 + vsub.f64 d16, d16, d0 + vstr d0, [sp, #8] + vsub.f64 d1, d16, d17 + vstr d1, [sp, #16] +.Lnmod3is1: + bl __kernel_cos + b .Lleave_sin +.Lxnottiny: + vmov.i64 d1, #0 + mov r0, #0 + bl __kernel_sin + b .Lleave_sin +.LxisNaN: + vsub.f64 d0, d0, d0 + b .Lleave_sin +.Lxge3pio4: + movt r2, #0x4139 + cmp r3, r2 + bgt .Lxgigantic + vmov_f64 d3, #0x60 + vldr d2, .Linvpio2 + vldr d18, .Lpio2_1 + vabs.f64 d16, d0 + vmla.f64 d3, d16, d2 + vcvt.s32.f64 s3, d3 + vcvt.f64.s32 d17, s3 + vmov r0, s3 + cmp r0, #31 + vmls.f64 d16, d17, d18 + vldr d18, .Lpio2_1t + vmul.f64 d18, d17, d18 + bgt .Lcomputeremainder + ldr r2, .Lnpio2_hw_ptr + sub lr, r0, #1 +.LPICnpio2_hw0: + add r12, pc, r2 + ldr r1, [r12, lr, lsl #2] + cmp r3, r1 + beq .Lcomputeremainder +.Lfinishthirditeration: + vsub.f64 d0, d16, d18 + vstr d0, [sp, #8] +.Lfinishcomputingremainder: + vsub.f64 d16, d16, d0 + cmp r4, #0 + vsub.f64 d1, d16, d18 + vstr d1, [sp, #16] + blt .Lhandlenegativex +.Lselectregion: + and r0, r0, #3 + cmp r0, #1 + beq .Lnmod3is1 + cmp r0, #2 + beq .Lnmod3is2 + cmp r0, #0 + bne .Lnmod3is0 + mov r0, #1 + bl __kernel_sin + b .Lleave_sin +.Lxgigantic: + asr r2, r3, #20 + vmov r6, r7, d0 + sub r2, r2, #1040 + mov r0, r6 + sub r2, r2, #6 + vldr d16, .Ltwo24 + sub r1, r3, r2, lsl #20 + vmov d18, r0, r1 + vcvt.s32.f64 s15, d18 + add r1, sp, #48 + mov r3, #3 + vcvt.f64.s32 d17, s15 + vsub.f64 d18, d18, d17 + vstr d17, [sp, #24] + vmul.f64 d18, d18, d16 + vcvt.s32.f64 s15, d18 + vcvt.f64.s32 d17, s15 + vsub.f64 d18, d18, d17 + vstr d17, [sp, #32] + vmul.f64 d16, d18, d16 + fcmpzd d16 + vstmdb r1!, {d16} + vmrs APSR_nzcv, fpscr + bne .Lprocessnonzeroterm +.Lskipzeroterms: + vldmdb r1!, {d16} + sub r3, r3, #1 + fcmpzd d16 + vmrs APSR_nzcv, fpscr + beq .Lskipzeroterms +.Lprocessnonzeroterm: + ldr r12, .Ltwo_over_pi_ptr + add r0, sp, #24 + add r1, sp, #8 +.LPICtwo_over_pi0: + add lr, pc, r12 + mov r12, #2 + str lr, [sp, #4] + str r12, [sp] + bl __kernel_rem_pio2 + cmp r4, #0 + vldr d0, [sp, #8] + blt .Lhandlenegativexalso + vldr d1, [sp, #16] + b .Lselectregion +.Lxnearpio2: + vldr d17, .Lpio2_2 + vsub.f64 d16, d16, d17 + vldr d17, .Lpio2_2t + b .Lfinalizesmallxremainder +.Lsmallxisnegative: + vldr d1, .Lpio2_1 + cmp r3, r2 + vadd.f64 d16, d0, d1 + beq .Lxnearnegpio2 + vldr d17, .Lpio2_1t +.Lfinalizesmallnegxremainder: + vadd.f64 d0, d16, d17 + vsub.f64 d16, d16, d0 + vstr d0, [sp, #8] + vadd.f64 d1, d16, d17 + vstr d1, [sp, #16] +.Lnmod3is0: + bl __kernel_cos + vneg.f64 d0, d0 + b .Lleave_sin +.Lnmod3is2: + mov r0, #1 + bl __kernel_sin + vneg.f64 d0, d0 + b .Lleave_sin +.Lcomputeremainder: + vsub.f64 d0, d16, d18 + asr r1, r3, #20 + vmov r2, r3, d0 + ubfx r3, r3, #20, #11 + rsb r3, r3, r1 + vstr d0, [sp, #8] + cmp r3, #16 + ble .Lfinishcomputingremainder + vldr d18, .Lpio2_2 + vmul.f64 d20, d17, d18 + vsub.f64 d19, d16, d20 + vsub.f64 d16, d16, d19 + vsub.f64 d18, d16, d20 + vldr d16, .Lpio2_2t + vnmls.f64 d18, d17, d16 + vsub.f64 d0, d19, d18 + vmov r2, r3, d0 + ubfx r3, r3, #20, #11 + rsb r1, r3, r1 + vstr d0, [sp, #8] + cmp r1, #49 + ble .Lfinishseconditeration + vldr d5, .Lpio2_3 + vmul.f64 d20, d17, d5 + vsub.f64 d16, d19, d20 + vsub.f64 d4, d19, d16 + vldr d19, .Lpio2_3t + vsub.f64 d18, d4, d20 + vnmls.f64 d18, d17, d19 + b .Lfinishthirditeration +.Lhandlenegativex: + vneg.f64 d0, d0 + rsb r0, r0, #0 + vneg.f64 d1, d1 + vstr d0, [sp, #8] + vstr d1, [sp, #16] + b .Lselectregion +.Lfinishseconditeration: + vmov d16, d19 + b .Lfinishcomputingremainder +.Lxnearnegpio2: + vldr d0, .Lpio2_2 + vldr d17, .Lpio2_2t + vadd.f64 d16, d16, d0 + b .Lfinalizesmallnegxremainder +.Lhandlenegativexalso: + vldr d6, [sp, #16] + vneg.f64 d0, d0 + rsb r0, r0, #0 + vneg.f64 d1, d6 + vstr d0, [sp, #8] + vstr d1, [sp, #16] + b .Lselectregion + +.align 3 +.Lpio2_1: + .word 0x54400000, 0x3ff921fb +.Lpio2_1t: + .word 0x1a626331, 0x3dd0b461 +.Linvpio2: + .word 0x6dc9c883, 0x3fe45f30 +.Ltwo24: + .word 0x00000000, 0x41700000 +.Lpio2_2: + .word 0x1a600000, 0x3dd0b461 +.Lpio2_2t: + .word 0x2e037073, 0x3ba3198a +.Lpio2_3: + .word 0x2e000000, 0x3ba3198a +.Lpio2_3t: + .word 0x252049c1, 0x397b839a +.Lnpio2_hw_ptr: + .word .Lnpio2_hw-(.LPICnpio2_hw0+8) +.Ltwo_over_pi_ptr: + .word .Ltwo_over_pi-(.LPICtwo_over_pi0+8) +END(sin) + + .section .rodata.npio2_hw,"a",%progbits + .align 2 +.Lnpio2_hw = . + 0 + .type npio2_hw, %object + .size npio2_hw, 128 +npio2_hw: + .word 0x3ff921fb + .word 0x400921fb + .word 0x4012d97c + .word 0x401921fb + .word 0x401f6a7a + .word 0x4022d97c + .word 0x4025fdbb + .word 0x402921fb + .word 0x402c463a + .word 0x402f6a7a + .word 0x4031475c + .word 0x4032d97c + .word 0x40346b9c + .word 0x4035fdbb + .word 0x40378fdb + .word 0x403921fb + .word 0x403ab41b + .word 0x403c463a + .word 0x403dd85a + .word 0x403f6a7a + .word 0x40407e4c + .word 0x4041475c + .word 0x4042106c + .word 0x4042d97c + .word 0x4043a28c + .word 0x40446b9c + .word 0x404534ac + .word 0x4045fdbb + .word 0x4046c6cb + .word 0x40478fdb + .word 0x404858eb + .word 0x404921fb + + .section .rodata.two_over_pi,"a",%progbits + .align 2 +.Ltwo_over_pi = . + 0 + .type two_over_pi, %object + .size two_over_pi, 264 +two_over_pi: + .word 0x00a2f983 + .word 0x006e4e44 + .word 0x001529fc + .word 0x002757d1 + .word 0x00f534dd + .word 0x00c0db62 + .word 0x0095993c + .word 0x00439041 + .word 0x00fe5163 + .word 0x00abdebb + .word 0x00c561b7 + .word 0x00246e3a + .word 0x00424dd2 + .word 0x00e00649 + .word 0x002eea09 + .word 0x00d1921c + .word 0x00fe1deb + .word 0x001cb129 + .word 0x00a73ee8 + .word 0x008235f5 + .word 0x002ebb44 + .word 0x0084e99c + .word 0x007026b4 + .word 0x005f7e41 + .word 0x003991d6 + .word 0x00398353 + .word 0x0039f49c + .word 0x00845f8b + .word 0x00bdf928 + .word 0x003b1ff8 + .word 0x0097ffde + .word 0x0005980f + .word 0x00ef2f11 + .word 0x008b5a0a + .word 0x006d1f6d + .word 0x00367ecf + .word 0x0027cb09 + .word 0x00b74f46 + .word 0x003f669e + .word 0x005fea2d + .word 0x007527ba + .word 0x00c7ebe5 + .word 0x00f17b3d + .word 0x000739f7 + .word 0x008a5292 + .word 0x00ea6bfb + .word 0x005fb11f + .word 0x008d5d08 + .word 0x00560330 + .word 0x0046fc7b + .word 0x006babf0 + .word 0x00cfbc20 + .word 0x009af436 + .word 0x001da9e3 + .word 0x0091615e + .word 0x00e61b08 + .word 0x00659985 + .word 0x005f14a0 + .word 0x0068408d + .word 0x00ffd880 + .word 0x004d7327 + .word 0x00310606 + .word 0x001556ca + .word 0x0073a8c9 + .word 0x0060e27b + .word 0x00c08c6b diff --git a/libm/upstream-freebsd/lib/msun/src/e_pow.c b/libm/upstream-freebsd/lib/msun/src/e_pow.c index 7607a4a72..e152628da 100644 --- a/libm/upstream-freebsd/lib/msun/src/e_pow.c +++ b/libm/upstream-freebsd/lib/msun/src/e_pow.c @@ -94,7 +94,11 @@ ivln2_h = 1.44269502162933349609e+00, /* 0x3FF71547, 0x60000000 =24b 1/ln2*/ ivln2_l = 1.92596299112661746887e-08; /* 0x3E54AE0B, 0xF85DDF44 =1/ln2 tail*/ double +#if defined(KRAIT_NEON_OPTIMIZATION) || defined(QCOM_NEON_OPTIMIZATION) +__full_ieee754_pow(double x, double y) +#else __ieee754_pow(double x, double y) +#endif { double z,ax,z_h,z_l,p_h,p_l; double y1,t1,t2,r,s,t,u,v,w; @@ -106,6 +110,35 @@ __ieee754_pow(double x, double y) EXTRACT_WORDS(hy,ly,y); ix = hx&0x7fffffff; iy = hy&0x7fffffff; +#if defined(KRAIT_NEON_OPTIMIZATION) || defined(QCOM_NEON_OPTIMIZATION) + + if (ly == 0) { + if (hy == ly) { + /* y==0.0, x**0 = 1 */ + return one; + } + else if (iy > 0x7ff00000) { + /* y is NaN, return x+y (NaN) */ + return x+y; + } + } + else if (iy >= 0x7ff00000) { + /* y is NaN, return x+y (NaN) */ + return x+y; + } + + if (lx == 0) { + if (ix > 0x7ff00000) { + /* x is NaN, return x+y (NaN) */ + return x+y; + } + } + else if (ix >= 0x7ff00000) { + /* x is NaN, return x+y (NaN) */ + return x+y; + } + +#else /* y==zero: x**0 = 1 */ if((iy|ly)==0) return one; @@ -116,7 +149,7 @@ __ieee754_pow(double x, double y) if(ix > 0x7ff00000 || ((ix==0x7ff00000)&&(lx!=0)) || iy > 0x7ff00000 || ((iy==0x7ff00000)&&(ly!=0))) return (x+0.0)+(y+0.0); - +#endif /* determine if y is an odd int when x < 0 * yisint = 0 ... y is not an integer * yisint = 1 ... y is an odd int diff --git a/libm/upstream-freebsd/lib/msun/src/k_cos.c b/libm/upstream-freebsd/lib/msun/src/k_cos.c index c4702e65b..6037e0dc0 100644 --- a/libm/upstream-freebsd/lib/msun/src/k_cos.c +++ b/libm/upstream-freebsd/lib/msun/src/k_cos.c @@ -68,6 +68,17 @@ C6 = -1.13596475577881948265e-11; /* 0xBDA8FAE9, 0xBE8838D4 */ double __kernel_cos(double x, double y) { +#if defined(KRAIT_NEON_OPTIMIZATION) || defined(QCOM_NEON_OPTIMIZATION) + double hz,z,zz,r,w,k; + + z = x*x; + zz = z*z; + k = x*y; + hz = (float)0.5*z; + r = z*(z*(C1+z*(C2+z*((C3+z*C4)+zz*(C5+z*C6))))); + w = one-hz; + return w + (((one-w)-hz) + (r-k)); +#else double hz,z,r,w; z = x*x; @@ -76,4 +87,5 @@ __kernel_cos(double x, double y) hz = 0.5*z; w = one-hz; return w + (((one-w)-hz) + (z*r-x*y)); +#endif } diff --git a/libm/upstream-freebsd/lib/msun/src/k_sin.c b/libm/upstream-freebsd/lib/msun/src/k_sin.c index 12ee8c143..afd2da897 100644 --- a/libm/upstream-freebsd/lib/msun/src/k_sin.c +++ b/libm/upstream-freebsd/lib/msun/src/k_sin.c @@ -59,6 +59,16 @@ S6 = 1.58969099521155010221e-10; /* 0x3DE5D93A, 0x5ACFD57C */ double __kernel_sin(double x, double y, int iy) { +#if defined(KRAIT_NEON_OPTIMIZATION) || defined(QCOM_NEON_OPTIMIZATION) + double z,zz,r,v; + + z = x*x; + zz = z*z; + v = z*x; + r = S2+z*((S3+z*S4)+zz*(S5+z*S6)); + if(iy==0) return x+v*(S1+z*r); + else return x-((z*(half*y-v*r)-y)-v*S1); +#else double z,r,v,w; z = x*x; @@ -67,4 +77,5 @@ __kernel_sin(double x, double y, int iy) v = z*x; if(iy==0) return x+v*(S1+z*r); else return x-((z*(half*y-v*r)-y)-v*S1); +#endif } diff --git a/libm/upstream-freebsd/lib/msun/src/math_private.h b/libm/upstream-freebsd/lib/msun/src/math_private.h index 8af2c650e..77520d778 100644 --- a/libm/upstream-freebsd/lib/msun/src/math_private.h +++ b/libm/upstream-freebsd/lib/msun/src/math_private.h @@ -723,6 +723,16 @@ irintl(long double x) #define __ieee754_remainderf remainderf #define __ieee754_scalbf scalbf +#if defined(KRAIT_NEON_OPTIMIZATION) || defined(QCOM_NEON_OPTIMIZATION) +int __kernel_rem_pio2(double*,double*,int,int,int) __attribute__((pcs("aapcs-vfp"))); +double __full_ieee754_pow(double,double); +#ifndef INLINE_REM_PIO2 +int __ieee754_rem_pio2(double,double*) __attribute__((pcs("aapcs-vfp"))); +#endif +double __kernel_sin(double,double,int) __attribute__((pcs("aapcs-vfp"))); +double __kernel_cos(double,double) __attribute__((pcs("aapcs-vfp"))); +double __kernel_tan(double,double,int) __attribute__((pcs("aapcs-vfp"))); +#else /* fdlibm kernel function */ int __kernel_rem_pio2(double*,double*,int,int,int); @@ -733,6 +743,8 @@ int __ieee754_rem_pio2(double,double*); double __kernel_sin(double,double,int); double __kernel_cos(double,double); double __kernel_tan(double,double,int); +#endif + double __ldexp_exp(double,int); #ifdef _COMPLEX_H double complex __ldexp_cexp(double complex,int); From 341ddb546559c316cbf3678af8487b542936fa9c Mon Sep 17 00:00:00 2001 From: Brent DeGraaf Date: Wed, 2 Oct 2013 09:47:11 -0400 Subject: [PATCH 12/89] libc: krait: Re-factor high-performance memcpy for thumb2 The majority of libc under bionic is built for thumb2. Refactor the high performance memcpy used in previous builds for thumb2, including information that can be used for stack-unwinding. Change-Id: I8a25e9cdc242a54f0fea41c135e11312f0bbb449 --- libc/arch-arm/krait/bionic/memcpy_base.S | 327 +++++++++++++++-------- 1 file changed, 209 insertions(+), 118 deletions(-) diff --git a/libc/arch-arm/krait/bionic/memcpy_base.S b/libc/arch-arm/krait/bionic/memcpy_base.S index 99fc255f3..068f2f60c 100644 --- a/libc/arch-arm/krait/bionic/memcpy_base.S +++ b/libc/arch-arm/krait/bionic/memcpy_base.S @@ -1,124 +1,215 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ +/*************************************************************************** + Copyright (c) 2009-2013 The Linux Foundation. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of The Linux Foundation nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + ***************************************************************************/ + +/* Assumes neon instructions and a cache line size of 64 bytes. */ +#include +#include /* - * This code assumes it is running on a processor that supports all arm v7 - * instructions, that supports neon instructions, and that has a 32 byte - * cache line. + * These default settings are good for all Krait-based systems + * as of this writing, but they can be overridden in: + * device///BoardConfig.mk + * by setting the following: + * TARGET_USE_KRAIT_BIONIC_OPTIMIZATION := true + * TARGET_USE_KRAIT_PLD_SET := true + * TARGET_KRAIT_BIONIC_PLDOFFS := + * TARGET_KRAIT_BIONIC_PLDSIZE := + * TARGET_KRAIT_BIONIC_PLDTHRESH := + * TARGET_KRAIT_BIONIC_BBTHRESH := */ -// Assumes neon instructions and a cache line size of 32 bytes. - -ENTRY_PRIVATE(MEMCPY_BASE) - .save {r0, lr} - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - - /* do we have at least 16-bytes to copy (needed for alignment below) */ - cmp r2, #16 - blo 5f - - /* align destination to cache-line for the write-buffer */ - rsb r3, r0, #0 - ands r3, r3, #0xF - beq 2f - - /* copy up to 15-bytes (count in r3) */ - sub r2, r2, r3 - movs ip, r3, lsl #31 - itt mi - ldrbmi lr, [r1], #1 - strbmi lr, [r0], #1 - itttt cs - ldrbcs ip, [r1], #1 - ldrbcs lr, [r1], #1 - strbcs ip, [r0], #1 - strbcs lr, [r0], #1 - movs ip, r3, lsl #29 - bge 1f - // copies 4 bytes, destination 32-bits aligned - vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! - vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! -1: bcc 2f - // copies 8 bytes, destination 64-bits aligned - vld1.8 {d0}, [r1]! - vst1.8 {d0}, [r0, :64]! - -2: /* make sure we have at least 64 bytes to copy */ - subs r2, r2, #64 - blo 2f - -1: /* The main loop copies 64 bytes at a time */ - vld1.8 {d0 - d3}, [r1]! - vld1.8 {d4 - d7}, [r1]! - pld [r1, #(32*8)] - subs r2, r2, #64 - vst1.8 {d0 - d3}, [r0, :128]! - vst1.8 {d4 - d7}, [r0, :128]! - bhs 1b - -2: /* fix-up the remaining count and make sure we have >= 32 bytes left */ - adds r2, r2, #32 - blo 4f - - /* Copy 32 bytes. These cache lines were already preloaded */ - vld1.8 {d0 - d3}, [r1]! - sub r2, r2, #32 - vst1.8 {d0 - d3}, [r0, :128]! - -4: /* less than 32 left */ - add r2, r2, #32 - tst r2, #0x10 - beq 5f - // copies 16 bytes, 128-bits aligned - vld1.8 {d0, d1}, [r1]! - vst1.8 {d0, d1}, [r0, :128]! - -5: /* copy up to 15-bytes (count in r2) */ - movs ip, r2, lsl #29 - bcc 1f - vld1.8 {d0}, [r1]! - vst1.8 {d0}, [r0]! -1: bge 2f - vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! - vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! -2: movs ip, r2, lsl #31 - itt mi - ldrbmi r3, [r1], #1 - strbmi r3, [r0], #1 - itttt cs - ldrbcs ip, [r1], #1 - ldrbcs lr, [r1], #1 - strbcs ip, [r0], #1 - strbcs lr, [r0], #1 - - ldmfd sp!, {r0, lr} - bx lr +#ifndef PLDOFFS +#define PLDOFFS (10) +#endif +#ifndef PLDTHRESH +#define PLDTHRESH (PLDOFFS) +#endif +#ifndef BBTHRESH +#define BBTHRESH (4096/64) +#endif +#if (PLDOFFS < 1) +#error Routine does not support offsets less than 1 +#endif +#if (PLDTHRESH < PLDOFFS) +#error PLD threshold must be greater than or equal to the PLD offset +#endif +#ifndef PLDSIZE +#define PLDSIZE (64) +#endif + .text + .fpu neon + +ENTRY(MEMCPY_BASE) +MEMCPY_BASE_ALIGNED: + // .cfi_startproc + .save {r0, r9, r10, lr} + // .cfi_def_cfa_offset 8 + //.cfi_rel_offset r0, 0 + //.cfi_rel_offset lr, 4 + cmp r2, #4 + blt .Lneon_lt4 + cmp r2, #16 + blt .Lneon_lt16 + cmp r2, #32 + blt .Lneon_16 + cmp r2, #64 + blt .Lneon_copy_32_a + + mov r12, r2, lsr #6 + cmp r12, #PLDTHRESH + ble .Lneon_copy_64_loop_nopld + + push {r9, r10} + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r9, 0 + .cfi_rel_offset r10, 4 + + cmp r12, #BBTHRESH + ble .Lneon_prime_pump + + add lr, r0, #0x400 + add r9, r1, #(PLDOFFS*PLDSIZE) + sub lr, lr, r9 + lsl lr, lr, #21 + lsr lr, lr, #21 + add lr, lr, #(PLDOFFS*PLDSIZE) + cmp r12, lr, lsr #6 + ble .Lneon_prime_pump + + itt gt + movgt r9, #(PLDOFFS) + rsbsgt r9, r9, lr, lsr #6 + ble .Lneon_prime_pump + + add r10, r1, lr + bic r10, #0x3F + + sub r12, r12, lr, lsr #6 + + cmp r9, r12 + itee le + suble r12, r12, r9 + movgt r9, r12 + movgt r12, #0 + + pld [r1, #((PLDOFFS-1)*PLDSIZE)] +.Lneon_copy_64_loop_outer_doublepld: + pld [r1, #((PLDOFFS)*PLDSIZE)] + vld1.32 {q0, q1}, [r1]! + vld1.32 {q2, q3}, [r1]! + ldr r3, [r10] + subs r9, r9, #1 + vst1.32 {q0, q1}, [r0]! + vst1.32 {q2, q3}, [r0]! + add r10, #64 + bne .Lneon_copy_64_loop_outer_doublepld + cmp r12, #0 + beq .Lneon_pop_before_nopld + + cmp r12, #(512*1024/64) + blt .Lneon_copy_64_loop_outer + +.Lneon_copy_64_loop_ddr: + vld1.32 {q0, q1}, [r1]! + vld1.32 {q2, q3}, [r1]! + pld [r10] + subs r12, r12, #1 + vst1.32 {q0, q1}, [r0]! + vst1.32 {q2, q3}, [r0]! + add r10, #64 + bne .Lneon_copy_64_loop_ddr + b .Lneon_pop_before_nopld + +.Lneon_prime_pump: + mov lr, #(PLDOFFS*PLDSIZE) + add r10, r1, #(PLDOFFS*PLDSIZE) + bic r10, #0x3F + sub r12, r12, #PLDOFFS + ldr r3, [r10, #(-1*PLDSIZE)] +.Lneon_copy_64_loop_outer: + vld1.32 {q0, q1}, [r1]! + vld1.32 {q2, q3}, [r1]! + ldr r3, [r10] + subs r12, r12, #1 + vst1.32 {q0, q1}, [r0]! + vst1.32 {q2, q3}, [r0]! + add r10, #64 + bne .Lneon_copy_64_loop_outer +.Lneon_pop_before_nopld: + mov r12, lr, lsr #6 + pop {r9, r10} + .cfi_restore r9 + .cfi_restore r10 + .cfi_adjust_cfa_offset -8 + +.Lneon_copy_64_loop_nopld: + vld1.32 {q8, q9}, [r1]! + vld1.32 {q10, q11}, [r1]! + subs r12, r12, #1 + vst1.32 {q8, q9}, [r0]! + vst1.32 {q10, q11}, [r0]! + bne .Lneon_copy_64_loop_nopld + ands r2, r2, #0x3f + .cfi_restore r0 + .cfi_adjust_cfa_offset -4 + beq .Lneon_exit +.Lneon_copy_32_a: + movs r3, r2, lsl #27 + bcc .Lneon_16 + vld1.32 {q0,q1}, [r1]! + vst1.32 {q0,q1}, [r0]! +.Lneon_16: + bpl .Lneon_lt16 + vld1.32 {q8}, [r1]! + vst1.32 {q8}, [r0]! + ands r2, r2, #0x0f + beq .Lneon_exit +.Lneon_lt16: + movs r3, r2, lsl #29 + itttt cs + ldrcs r3, [r1], #4 + strcs r3, [r0], #4 + ldrcs r3, [r1], #4 + strcs r3, [r0], #4 + itt mi + ldrmi r3, [r1], #4 + strmi r3, [r0], #4 +.Lneon_lt4: + movs r2, r2, lsl #31 + itt cs + ldrhcs r3, [r1], #2 + strhcs r3, [r0], #2 + itt mi + ldrbmi r3, [r1] + strbmi r3, [r0] +.Lneon_exit: + pop {r0, lr} + bx lr + //.cfi_endproc END(MEMCPY_BASE) + From ae62d8384a903217af343d1e4114bc73c9fd1a5d Mon Sep 17 00:00:00 2001 From: Biswajit Paul Date: Mon, 29 Jul 2013 16:48:31 +0530 Subject: [PATCH 13/89] bionic: Detect userspace memory leak When process is consuming memory beyond the limit specified by property, it will print all the allocations of the same process. To enable this feature, do following steps: 1. libc.debug.malloc 40 2. libc.debug.malloc.program 3. libc.debug.malloc.maxprocsize 4. libc.malloc.minalloclim Conflicts: libc/bionic/malloc_debug_check.cpp libc/bionic/malloc_debug_common.cpp Change-Id: I03a4de9643ec954802b26443ce5685975ea30f89 --- libc/bionic/malloc_debug_check.cpp | 143 +++++++++++++++++++++++++++- libc/bionic/malloc_debug_common.cpp | 23 +++++ 2 files changed, 164 insertions(+), 2 deletions(-) diff --git a/libc/bionic/malloc_debug_check.cpp b/libc/bionic/malloc_debug_check.cpp index dee03fae1..da48059db 100644 --- a/libc/bionic/malloc_debug_check.cpp +++ b/libc/bionic/malloc_debug_check.cpp @@ -45,6 +45,7 @@ #include #include #include +#include #include "debug_mapinfo.h" #include "debug_stacktrace.h" @@ -55,6 +56,14 @@ #include "private/libc_logging.h" #include "private/ScopedPthreadMutexLocker.h" +extern unsigned int malloc_sig_enabled; +extern unsigned int min_allocation_report_limit; +extern unsigned int max_allocation_limit; +extern char* process_name; +static size_t total_count = 0; +static bool isDumped = false; +static bool sigHandled = false; + #define MAX_BACKTRACE_DEPTH 16 #define ALLOCATION_TAG 0x1ee7d00d #define BACKLOG_TAG 0xbabecafe @@ -63,6 +72,10 @@ #define FRONT_GUARD_LEN (1<<5) #define REAR_GUARD 0xbb #define REAR_GUARD_LEN (1<<5) +#define FRONT_GUARD_SS 0xab + +static void malloc_sigaction(int signum, siginfo_t * sg, void * cxt); +static struct sigaction default_sa; static void log_message(const char* format, ...) { va_list args; @@ -135,9 +148,14 @@ static inline void init_front_guard(hdr_t* hdr) { memset(hdr->front_guard, FRONT_GUARD, FRONT_GUARD_LEN); } +static inline void set_snapshot(hdr_t* hdr) { + memset(hdr->front_guard, FRONT_GUARD_SS, FRONT_GUARD_LEN); +} + static inline bool is_front_guard_valid(hdr_t* hdr) { for (size_t i = 0; i < FRONT_GUARD_LEN; i++) { - if (hdr->front_guard[i] != FRONT_GUARD) { + if (!((hdr->front_guard[i] == FRONT_GUARD) || + (hdr->front_guard[i] == FRONT_GUARD_SS))) { return false; } } @@ -171,6 +189,9 @@ static inline bool is_rear_guard_valid(hdr_t* hdr) { } static inline void add_locked(hdr_t* hdr, hdr_t** tail, hdr_t** head) { + if (hdr->tag == ALLOCATION_TAG) { + total_count += hdr->size; + } hdr->prev = NULL; hdr->next = *head; if (*head) @@ -181,6 +202,9 @@ static inline void add_locked(hdr_t* hdr, hdr_t** tail, hdr_t** head) { } static inline int del_locked(hdr_t* hdr, hdr_t** tail, hdr_t** head) { + if (hdr->tag == ALLOCATION_TAG) { + total_count -= hdr->size; + } if (hdr->prev) { hdr->prev->next = hdr->next; } else { @@ -194,6 +218,25 @@ static inline int del_locked(hdr_t* hdr, hdr_t** tail, hdr_t** head) { return 0; } +static void snapshot_report_leaked_nodes() { + log_message("%s: %s\n", __FILE__, __FUNCTION__); + hdr_t * iterator = head; + size_t total_size = 0; + do { + if (iterator->front_guard[0] == FRONT_GUARD && + iterator->size >= min_allocation_report_limit) { + log_message("obj %p, size %d", iterator, iterator->size); + total_size += iterator->size; + log_backtrace(iterator->bt, iterator->bt_depth); + log_message("------------------------------"); // as an end marker + // Marking the node as we do not want to print it again. + set_snapshot(iterator); + } + iterator = iterator->next; + } while (iterator); + log_message("Total Pending allocations after last snapshot: %d", total_size); +} + static inline void add(hdr_t* hdr, size_t size) { ScopedPthreadMutexLocker locker(&lock); hdr->tag = ALLOCATION_TAG; @@ -202,6 +245,21 @@ static inline void add(hdr_t* hdr, size_t size) { init_rear_guard(hdr); ++g_allocated_block_count; add_locked(hdr, &tail, &head); + if (total_count >= max_allocation_limit && !isDumped && malloc_sig_enabled) { + isDumped = true; + log_message("Maximum limit of the %s process (%d Bytes) size has reached."\ + "Maximum limit is set to:%d Bytes\n", process_name, + total_count, max_allocation_limit); + log_message("Start dumping allocations of the process %s", process_name); + log_message("+++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++ ***\n"); + + // Print allocations of the process + snapshot_report_leaked_nodes(); + + log_message("*** +++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++\n"); + log_message("Completed dumping allocations of the process %s", process_name); + + } } static inline int del(hdr_t* hdr) { @@ -233,7 +291,8 @@ static bool was_used_after_free(hdr_t* hdr) { static inline int check_guards(hdr_t* hdr, int* safe) { *safe = 1; if (!is_front_guard_valid(hdr)) { - if (hdr->front_guard[0] == FRONT_GUARD) { + if ((hdr->front_guard[0] == FRONT_GUARD) || + ((hdr->front_guard[0] == FRONT_GUARD_SS))) { log_message("+++ ALLOCATION %p SIZE %d HAS A CORRUPTED FRONT GUARD\n", user(hdr), hdr->size); } else { @@ -637,6 +696,8 @@ static void ReportMemoryLeaks() { pthread_key_t g_debug_calls_disabled; +#define DEBUG_SIGNAL SIGWINCH + extern "C" bool malloc_debug_initialize(HashTable* hash_table, const MallocDebug* malloc_dispatch) { g_hash_table = hash_table; g_malloc_dispatch = malloc_dispatch; @@ -656,6 +717,26 @@ extern "C" bool malloc_debug_initialize(HashTable* hash_table, const MallocDebug __libc_format_log(ANDROID_LOG_INFO, "libc", "not gathering backtrace information\n"); } +/* Initializes malloc debugging framework. + * See comments on MallocDebugInit in malloc_debug_common.h + */ + if (malloc_sig_enabled) { + struct sigaction sa; //local or static? + //struct sigaction sa_snapshot; //local or static? + sa.sa_handler = NULL; + sa.sa_sigaction = malloc_sigaction; + sigemptyset(&sa.sa_mask); + sigaddset(&sa.sa_mask, DEBUG_SIGNAL); + sa.sa_flags = SA_SIGINFO; + sa.sa_restorer = NULL; + if (sigaction(DEBUG_SIGNAL, &sa, &default_sa) < 0) { + log_message("Failed to register signal handler w/ errno %s", strerror(errno)); + malloc_sig_enabled = 0; + } else { + log_message("Registered signal handler"); + sigHandled = false; + } + } if (g_backtrace_enabled) { backtrace_startup(); } @@ -668,9 +749,67 @@ extern "C" void malloc_debug_finalize(int malloc_debug_level) { if (malloc_debug_level == 10) { ReportMemoryLeaks(); } + if (malloc_sig_enabled) { + log_message("Deregister %d signal handler", DEBUG_SIGNAL); + sigaction(DEBUG_SIGNAL, &default_sa, NULL); + malloc_sig_enabled = 0; + sigHandled = false; + } if (g_backtrace_enabled) { backtrace_shutdown(); } pthread_setspecific(g_debug_calls_disabled, NULL); } + +static void snapshot_nodes_locked() { + log_message("%s: %s\n", __FILE__, __FUNCTION__); + hdr_t * iterator = head; + do { + if (iterator->front_guard[0] == FRONT_GUARD) { + set_snapshot(iterator); + } + iterator = iterator->next; + } while (iterator); +} + +static void malloc_sigaction(int signum, siginfo_t * info, void * context) +{ + log_message("%s: %s\n", __FILE__, __FUNCTION__); + log_message("%s got %d signal from PID: %d (context:%x)\n", + __func__, signum, info->si_pid, context); + + if (signum != DEBUG_SIGNAL) { + log_message("RECEIVED %d instead of %d\n", signum, DEBUG_SIGNAL); + return; + } + + ScopedPthreadMutexLocker locker(&lock); + + log_message("Process under observation:%s", process_name); + log_message("Maximum process size limit:%d Bytes", max_allocation_limit); + log_message("Won't print allocation below %d Bytes", min_allocation_report_limit); + log_message("Total count: %d\n", total_count); + + if (!head) { + log_message("No allocations?"); + return; + } + // If sigHandled is false, meaning it's being handled first time + if (!sigHandled) { + sigHandled = true; + // Marking the nodes assuming that they should not be leaked nodes. + snapshot_nodes_locked(); + } else { + // We need to print new allocations now + log_message("Start dumping allocations of the process %s", process_name); + log_message("+++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++ ***\n"); + + // Print allocations of the process + snapshot_report_leaked_nodes(); + + log_message("*** +++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++ *** +++\n"); + log_message("Completed dumping allocations of the process %s", process_name); + } + return; +} diff --git a/libc/bionic/malloc_debug_common.cpp b/libc/bionic/malloc_debug_common.cpp index 0b6a1421b..e0d439acf 100644 --- a/libc/bionic/malloc_debug_common.cpp +++ b/libc/bionic/malloc_debug_common.cpp @@ -295,6 +295,11 @@ extern "C" void* valloc(size_t bytes) { #include #include "private/libc_logging.h" +unsigned int malloc_sig_enabled = 0; +unsigned int max_allocation_limit; +unsigned int min_allocation_report_limit; +const char* process_name; + template static void InitMallocFunction(void* malloc_impl_handler, FunctionType* func, const char* prefix, const char* suffix) { char symbol[128]; @@ -396,6 +401,21 @@ static void malloc_init_impl() { } so_name = "libc_malloc_debug_qemu.so"; break; + case 40: + malloc_sig_enabled = 1; + char debug_proc_size[PROP_VALUE_MAX]; + if (__system_property_get("libc.debug.malloc.maxprocsize", debug_proc_size)) + max_allocation_limit = atoi(debug_proc_size); + else + max_allocation_limit = 30 * 1024 * 1024; // In Bytes [Default is 30 MB] + if (__system_property_get("libc.debug.malloc.minalloclim", debug_proc_size)) + min_allocation_report_limit = atoi(debug_proc_size); + else + min_allocation_report_limit = 10 * 1024; // In Bytes [Default is 10 KB] + process_name = getprogname(); + + so_name = "libc_malloc_debug_leak.so"; + break; default: error_log("%s: Debug level %d is unknown\n", getprogname(), g_malloc_debug_level); return; @@ -456,6 +476,9 @@ static void malloc_init_impl() { case 20: InitMalloc(malloc_impl_handle, &malloc_dispatch_table, "qemu_instrumented"); break; + case 40: + InitMalloc(malloc_impl_handle, &malloc_dispatch_table, "chk"); + break; default: break; } From 9aa974c6d69c8eaf2355d86c64c27f1dc30d6ac0 Mon Sep 17 00:00:00 2001 From: Brent DeGraaf Date: Fri, 19 Sep 2014 09:27:24 -0400 Subject: [PATCH 14/89] bionic/libm pow optimizations for arm64 Optimize a narrow, commonly used range of inputs for faster pow. Change-Id: Ic2f2aefe73433fb354429f5a9de5141b716ca89b --- libm/Android.mk | 7 +- libm/arm64/e_pow64.S | 450 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 456 insertions(+), 1 deletion(-) create mode 100644 libm/arm64/e_pow64.S diff --git a/libm/Android.mk b/libm/Android.mk index 77ec5d249..30c2bc7fc 100644 --- a/libm/Android.mk +++ b/libm/Android.mk @@ -257,10 +257,14 @@ ifeq ($(TARGET_USE_QCOM_BIONIC_OPTIMIZATION),true) libm_arm_includes += $(LOCAL_PATH)/../libc/ libm_arm64_src_files += \ + arm64/e_pow64.S \ upstream-freebsd/lib/msun/src/s_cos.c \ upstream-freebsd/lib/msun/src/s_sin.c \ upstream-freebsd/lib/msun/src/e_sqrtf.c \ upstream-freebsd/lib/msun/src/e_sqrt.c + + libm_arm64_cflags += -DQCOM_NEON_OPTIMIZATION + libm_arm64_includes += $(LOCAL_PATH)/../libc/ else libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_cos.c \ @@ -286,7 +290,8 @@ LOCAL_CFLAGS_arm := $(libm_arm_cflags) LOCAL_C_INCLUDES_arm := $(LOCAL_PATH)/arm $(libm_arm_includes) LOCAL_SRC_FILES_arm := arm/fenv.c $(libm_arm_src_files) -LOCAL_C_INCLUDES_arm64 := $(libm_ld_includes) +LOCAL_CFLAGS_arm64 := $(libm_arm64_cflags) +LOCAL_C_INCLUDES_arm64 := $(libm_ld_includes) $(libm_arm64_includes) LOCAL_SRC_FILES_arm64 := arm64/fenv.c $(libm_ld_src_files) $(libm_arm64_src_files) LOCAL_C_INCLUDES_x86 := $(LOCAL_PATH)/i387 diff --git a/libm/arm64/e_pow64.S b/libm/arm64/e_pow64.S new file mode 100644 index 000000000..2a30c4aaf --- /dev/null +++ b/libm/arm64/e_pow64.S @@ -0,0 +1,450 @@ +/* Copyright (c) 2009-2014 The Linux Foundation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Linux Foundation nor the names of its contributors may + * be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +//#define NO_FUSED_MULTIPLY + +#define KRAIT_NO_AAPCS_VFP_MODE + +ENTRY(pow) +#if defined(KRAIT_NO_AAPCS_VFP_MODE) + // ARM ABI has inputs coming in via d registers, lets copy to x registers + fmov x0, d0 + fmov x1, d1 +#endif + mov w12, #0x40100000 // high word of 64-bit 4.0 + + // pre-staged bp values + ldr d5, .LbpA + ldr d3, .LbpB + + // load two fifths into constant term in case we need it due to offsets + ldr d6, .Ltwofifths + + // bp is initially 1.0, may adjust later based on x value + fmov d4, #1.0 + + // twoto1o5 = 2^(1/5) (input bracketing) + ldr x4, .Ltwoto1o5 + + // twoto3o5 = 2^(3/5) (input bracketing) + ldr x5, .Ltwoto3o5 + + // extract xmantissa + bic x6, x0, #0xFFF0000000000000 + + // begin preparing a mask for normalization (high 32-bit mask) + movi d31, #0xFFFFFFFF00000000 + + // double_1 = (double) 1.0 + fmov d28, #1.0 + + cmp x6, x4 + + shl d30, d31, #20 // d30 can mask just sign/exp bits + ushr d29, d31, #63 // mask has only bit 1 set + + adr x10, .LliteralTable // x10->k4 in literal table (below) + + bhi .Lxgt2to1over5 + // zero out lg2 constant term if don't offset our input + fsub d6, d6, d6 + b .Lxle2to1over5 + +.Lxgt2to1over5: + // if normalized x > 2^(1/5), bp = 1 + (2^(2/5)-1) = 2^(2/5) + fadd d4, d4, d5 + +.Lxle2to1over5: + ldr d5, .Lln2 // d5 = ln2 = 0.69314718056 + + cmp x6, x5 // non-normalized compare + +//@@@ X Value Normalization @@@@ + + // ss = abs(x) 2^(-1024) + bic v16.8B, v0.8B, v30.8B // mantissa of x into v16.8B (aka d16) + + // N = (floor(log2(x)) + 0x3ff) * 2^52 + and v2.8B, v0.8B, v30.8B // exponent of x (d0) into v2.8B aka d2 + + bls .Lxle2to3over5 // branch not taken if (x6 > x5) + // if normalized x > 2^(3/5), bp = 2^(2/5) + (2^(4/5) - 2^(2/5)) = 2^(4/5) + fadd d4, d4, d3 // d4 = 2^(2/5) + (2^(4/5) - 2^(2/5)) = 2^(4/5) + fadd d6, d6, d6 // d6 = 2/5 + 2/5 = 4/5 or 0 (see logic above) + +.Lxle2to3over5: + + lsr x2, x0, #32 // Need just high word of x...x2 can take it + cmp w2, w12 // Compare x to 4.0 (high word only) + lsr x3, x1, #32 // Need just high word of y...x3 can take it. + ccmp w3, w12, #2, ls // If x < 4.0, compare y to 4.0 (high word) + bic w12, w12, #0xFFFF0000 // Change w12 for compare to 0.0000325 + orr w12, w12, #0x3fe00000 + ccmp w12, w2, #2, ls // If y < 4, compare 0.5 to x + + // load log2 polynomial series constants + ldp d24, d25, [x10, #0] + ldp d26, d27, [x10, #16] + + // s = abs(x) 2^(-floor(log2(x))) (normalize abs(x) to around 1) + orr v16.8B, v16.8B, v28.8B + +//@@@ 3/2 (Log(bp(1+s)/(1-s))) input computation (s = (x-bp)/(x+bp)) @@@@ + + fsub d19, d16, d4 // take normalized x and subtract 2^(4/5) from it + fadd d20, d16, d4 + bhi .LuseFullImpl // |x| < 0.5 or x > 4 or y > 4 + + // s = (x-1)/(x+1) + fdiv d16, d19, d20 + + // load 2/(3log2) into lg2coeff + ldr d21, .Ltwooverthreeln2 + + // N = floor(log2(x)) * 2^52 + sub d2, d2, d28 + +//@@@ 3/2 (Log(bp(1+s)/(1-s))) polynomial series @@@@ + + // ss2 = ((x-bp)/(x+bp))^2 + fmul d17, d16, d16 + + // ylg2x = 3.0 + fmov d0, #3.0 + fmul d18, d17, d17 + + // todo: useful later for two-way clamp + fmul d21, d21, d1 + + // N = floor(log2(x)) + sshr d2, d2, #52 + // k3 = ss^2 * L4 + L3 +#ifdef NO_FUSED_MULTIPLY + fmul d3, d17, v24.2D[0] + fadd d25, d25, d3 + + // k1 = ss^2 * L2 + L1 + fmul d3, d17, v26.2D[0] + fadd d27, d27, d3 +#else + fmla d25, d17, v24.2D[0] + + // k1 = ss^2 * L2 + L1 + fmla d27, d17, v26.2D[0] +#endif + + // scale ss by 2/(3 ln 2) + fmul d21, d16, d21 + + // ylg2x = 3.0 + s^2 + fadd d0, d0, d17 + + fmov x2, d2 + scvtf d3, w2 // Low-order 32-bit integer half of d2 to fp64 + + // k1 = s^4 (s^2 L4 + L3) + s^2 L2 + L1 +#ifdef NO_FUSED_MULTIPLY + fmul d31, d18, v25.2D[0] + fadd d27, d27, d31 +#else + fmla d27, d18, v25.2D[0] +#endif + // add in constant term + fadd d3, d3, d6 + + // ylg2x = 3.0 + s^2 + s^4 (s^4 (s^2 L4 + L3) + s^2 L2 + L1) +#ifdef NO_FUSED_MULTIPLY + fmul d31, d18, v27.2D[0] + fadd d0, d0, d31 +#else + fmla d0, d18, v27.2D[0] +#endif + // ylg2x = y 2 s / (3 ln(2)) (3.0 + s^2 + s^4 (s^4(s^2 L4 + L3) + s^2 L2 + L1) + fmul d0, d21, d0 + +//@@@ Compute input to Exp(s) (s = y(n + log2(x)) - (floor(8 yn + 1)/8 + floor(8 ylog2(x) + 1)/8) @@@@@ + + // mask to extract bit 1 (2^-2 from our fixed-point representation) + shl d4, d29, #1 + + // double_n = y * n + fmul d3, d3, d1 + + // Load 2^(1/4) for later computations + ldr d6, .Ltwoto1o4 + + // either add or subtract one based on the sign of double_n and ylg2x + sshr d16, d0, #62 + sshr d19, d3, #62 + + // move unmodified y*lg2x into temp space + fmov d17, d0 + + // compute floor(8 y * n + 1)/8 + // and floor(8 y (log2(x)) + 1)/8 + fcvtzs w2, d0, #3 // no instruction exists to use s0 as a direct target + fmov s0, w2 // run our conversion into w2, then mov it to compensate + // move unmodified y*n into temp space + fmov d18, d3 + fcvtzs w2, d3, #3 + fmov s3, w2 + + // load exp polynomial series constants + ldp d20, d21, [x10, #32] + ldp d22, d23, [x10, #48] + ldp d24, d25, [x10, #64] + ldp d26, d27, [x10, #80] + + // mask to extract bit 2 (2^-1 from our fixed-point representation) + shl d1, d29, #2 + + // make rounding offsets either 1 or -1 instead of 0 or -2 + orr v16.8B, v16.8B, v29.8B + orr v19.8B, v19.8B, v29.8B + + // round up to the nearest 1/8th + add d0, d0, d16 + add d3, d3, d19 + + // clear out round-up bit for y log2(x) + bic v0.8B, v0.8B, v29.8B + // clear out round-up bit for yn + bic v3.8B, v3.8B, v29.8B + // add together the (fixed precision) rounded parts + add d31, d3, d0 + // turn int_n into a double with value 2^int_n + shl d2, d31, #49 + // compute masks for 2^(1/4) and 2^(1/2) fixups for fractional part of fixed-precision rounded values: + and v4.8B, v4.8B, v31.8B + and v1.8B, v1.8B, v31.8B + + // convert back into floating point, d3 now holds (double) floor(8 y * n + 1)/8 + // d0 now holds (double) floor(8 y * log2(x) + 1)/8 + fmov w2, s0 + scvtf d0, w2, #3 + fmov w2, s3 + scvtf d3, w2, #3 + + // put the 2 bit (0.5) through the roof of twoto1o2mask (make it 0x0 or 0xffffffffffffffff) + uqshl d1, d1, #62 + + // put the 1 bit (0.25) through the roof of twoto1o4mask (make it 0x0 or 0xffffffffffffffff) + uqshl d4, d4, #63 + + // center y*log2(x) fractional part between -0.125 and 0.125 by subtracting (double) floor(8 y * log2(x) + 1)/8 + fsub d17, d17, d0 + // center y*n fractional part between -0.125 and 0.125 by subtracting (double) floor(8 y * n + 1)/8 + fsub d18, d18, d3 + + // Add fractional parts of yn and y log2(x) together + fadd d16, d17, d18 + + // Result = 1.0 (offset for exp(s) series) + fmov d0, #1.0 + + // multiply fractional part of y * log2(x) by ln(2) + fmul d16, d5, d16 + +//@@@ 10th order polynomial series for Exp(s) @@@@ + + // ss2 = (ss)^2 + fmul d17, d16, d16 + + // twoto1o2mask = twoto1o2mask & twoto1o4 + and v1.8B, v1.8B, v6.8B + // twoto1o2mask = twoto1o2mask & twoto1o4 + and v4.8B, v4.8B, v6.8B + + // Result = 1.0 + ss + fadd d0, d0, d16 + + // k7 = ss k8 + k7 +#ifdef NO_FUSED_MULTIPLY + fmul d31, d16, v20.2D[0] + fadd d21, d21, d31 +#else + fmla d21, d16, v20.2D[0] +#endif + // ss4 = (ss*ss) * (ss*ss) + fmul d18, d17, d17 + + // twoto1o2mask = twoto1o2mask | (double) 1.0 - results in either 1.0 or 2^(1/4) in twoto1o2mask + orr v1.8B, v1.8B, v28.8B + // twoto1o2mask = twoto1o4mask | (double) 1.0 - results in either 1.0 or 2^(1/4) in twoto1o4mask + orr v4.8B, v4.8B, v28.8B + + // sign could be set up here, but for now expadjustment = 1.0 + fmov d7, #1.0 + + // ss3 = (ss*ss) * ss + fmul d19, d17, d16 + + // k0 = 1/2 (first non-unity coefficient) + fmov d28, #0.5 + + // Mask out non-exponent bits to make sure we have just 2^int_n + and v2.8B, v2.8B, v30.8B + + // square twoto1o2mask to get 1.0 or 2^(1/2) + fmul d1, d1, d1 + + // multiply twoto2o4mask into the exponent output adjustment value + fmul d7, d7, d4 + +#ifdef NO_FUSED_MULTIPLY + // k5 = ss k6 + k5 + fmul d31, d16, v22.2D[0] + fadd d23, d23, d31 + + // k3 = ss k4 + k3 + fmul d31, d16, v24.2D[0] + fadd d25, d25, d31 + + // k1 = ss k2 + k1 + fmul d31, d16, v26.2D[0] + fadd d27, d27, d31 +#else + // k5 = ss k6 + k5 + fmla d23, d16, v22.2D[0] + + // k3 = ss k4 + k3 + fmla d25, d16, v24.2D[0] + + // k1 = ss k2 + k1 + fmla d27, d16, v26.2D[0] +#endif + // multiply twoto1o2mask into exponent output adjustment value + fmul d7, d7, d1 +#ifdef NO_FUSED_MULTIPLY + // k5 = ss^2 ( ss k8 + k7 ) + ss k6 + k5 + fmul d31, d17, v21.2D[0] + fadd d23, d23, d31 + + // k1 = ss^2 ( ss k4 + k3 ) + ss k2 + k1 + fmul d31, d17, v25.2D[0] + fadd d27, d27, d31 + + // Result = 1.0 + ss + 1/2 ss^2 + fmul d31, d17, v28.2D[0] + fadd d0, d0, d31 +#else + // k5 = ss^2 ( ss k8 + k7 ) + ss k6 + k5 + fmla d23, d17, v21.2D[0] + + // k1 = ss^2 ( ss k4 + k3 ) + ss k2 + k1 + fmla d27, d17, v25.2D[0] + + // Result = 1.0 + ss + 1/2 ss^2 + fmla d0, d17, v28.2D[0] +#endif + // Adjust int_n so that it's a double precision value that can be multiplied by Result + add d7, d2, d7 +#ifdef NO_FUSED_MULTIPLY + // k1 = ss^4 ( ss^2 ( ss k8 + k7 ) + ss k6 + k5 ) + ss^2 ( ss k4 + k3 ) + ss k2 + k1 + fmul d31, d18, v23.2D[0] + fadd d27, d27, d31 + + // Result = 1.0 + ss + 1/2 ss^2 + ss^3 ( ss^4 ( ss^2 ( ss k8 + k7 ) + ss k6 + k5 ) + ss^2 ( ss k4 + k3 ) + ss k2 + k1 ) + fmul d31, d19, v27.2D[0] + fadd d0, d0, d31 +#else + // k1 = ss^4 ( ss^2 ( ss k8 + k7 ) + ss k6 + k5 ) + ss^2 ( ss k4 + k3 ) + ss k2 + k1 + fmla d27, d18, v23.2D[0] + + // Result = 1.0 + ss + 1/2 ss^2 + ss^3 ( ss^4 ( ss^2 ( ss k8 + k7 ) + ss k6 + k5 ) + ss^2 ( ss k4 + k3 ) + ss k2 + k1 ) + fmla d0, d19, v27.2D[0] +#endif + // multiply by adjustment (sign*(rounding ? sqrt(2) : 1) * 2^int_n) + fmul d0, d7, d0 + +.LleavePow: +#if defined(KRAIT_NO_AAPCS_VFP_MODE) + // return Result (FP) + // fmov x0, d0 +#endif +.LleavePowDirect: + // leave directly returning whatever is in d0 + ret +.LuseFullImpl: + fmov d0, x0 + fmov d1, x1 + b __full_ieee754_pow + +.align 6 +.LliteralTable: +// Least-sqares tuned constants for 11th order (log2((1+s)/(1-s)): +.LL4: // ~3/11 + .long 0x53a79915, 0x3fd1b108 +.LL3: // ~1/3 + .long 0x9ca0567a, 0x3fd554fa +.LL2: // ~3/7 + .long 0x1408e660, 0x3fdb6db7 +.LL1: // ~3/5 + .long 0x332D4313, 0x3fe33333 + +// Least-squares tuned constants for 10th order exp(s): +.LE10: // ~1/3628800 + .long 0x25c7ba0a, 0x3e92819b +.LE9: // ~1/362880 + .long 0x9499b49c, 0x3ec72294 +.LE8: // ~1/40320 + .long 0xabb79d95, 0x3efa019f +.LE7: // ~1/5040 + .long 0x8723aeaa, 0x3f2a019f +.LE6: // ~1/720 + .long 0x16c76a94, 0x3f56c16c +.LE5: // ~1/120 + .long 0x11185da8, 0x3f811111 +.LE4: // ~1/24 + .long 0x5555551c, 0x3fa55555 +.LE3: // ~1/6 + .long 0x555554db, 0x3fc55555 + +.LbpA: // (2^(2/5) - 1) + .long 0x4ee54db1, 0x3fd472d1 + +.LbpB: // (2^(4/5) - 2^(2/5)) + .long 0x1c8a36cf, 0x3fdafb62 + +.Ltwofifths: // 2/5 + .long 0x9999999a, 0x3fd99999 + +.Ltwooverthreeln2: + .long 0xDC3A03FD, 0x3FEEC709 + +.Ltwoto1o5: // 2^(1/5) exponent 3ff stripped for non-normalized compares + .long 0x86BAE675, 0x00026111 + +.Ltwoto3o5: // 2^(3/5) exponent 3ff stripped for non-normalized compares + .long 0x03B2AE5C, 0x00084060 + +.Lln2: // ln(2) + .long 0xFEFA39EF, 0x3FE62E42 + +.Ltwoto1o4: // 2^1/4 + .long 0x0a31b715, 0x3ff306fe +END(pow) From 7b93f5af2fd5ee9bca6bee5f5bbb2cc33cb56deb Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Tue, 7 Oct 2014 11:58:22 -0700 Subject: [PATCH 15/89] libm: add missing symbols to optimized math functions add the following missing symbols to optimized math functions: sqrtl => sqrt sinl => sin cosl => cos these symbols will be weak references to the functions at right-hand side when compiler internal macro (__LDBL_MANT_DIG__ == 53) is true Change-Id: Ie3b3e720b3207bbc13574b9b6d42738767f22754 --- libm/arm/e_sqrt.S | 5 +++++ libm/arm/s_cos.S | 10 ++++++++-- libm/arm/s_sin.S | 10 ++++++++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/libm/arm/e_sqrt.S b/libm/arm/e_sqrt.S index f2c88ff98..0b5b10c79 100644 --- a/libm/arm/e_sqrt.S +++ b/libm/arm/e_sqrt.S @@ -34,3 +34,8 @@ ENTRY(sqrt) vmov.f64 r0, r1, d0 bx lr END(sqrt) + +#if (__LDBL_MANT_DIG__ == 53) +.weak sqrtl +.equ sqrtl, sqrt +#endif diff --git a/libm/arm/s_cos.S b/libm/arm/s_cos.S index eae161e5a..5c8183bb5 100644 --- a/libm/arm/s_cos.S +++ b/libm/arm/s_cos.S @@ -50,8 +50,6 @@ #define vmov_f64 fconstd -ENTRY(cosl) -END(cosl) ENTRY(cos) push {r4, r6, r7, lr} vmov d0, r0, r1 @@ -308,6 +306,14 @@ ENTRY(cos) .word .Ltwo_over_pi-(.LPICtwo_over_pi0+8) END(cos) +#if (__LDBL_MANT_DIG__ == 53) +.weak cosl +.equ cosl, cos +#else +ENTRY(cosl) +END(cosl) +#endif + .section .rodata.npio2_hw,"a",%progbits .align 2 .Lnpio2_hw = . + 0 diff --git a/libm/arm/s_sin.S b/libm/arm/s_sin.S index 3c449154b..8b9eae4ed 100644 --- a/libm/arm/s_sin.S +++ b/libm/arm/s_sin.S @@ -50,8 +50,6 @@ #define vmov_f64 fconstd -ENTRY(sinl) -END(sinl) ENTRY(sin) push {r4, r6, r7, lr} vmov d0, r0, r1 @@ -303,6 +301,14 @@ ENTRY(sin) .word .Ltwo_over_pi-(.LPICtwo_over_pi0+8) END(sin) +#if (__LDBL_MANT_DIG__ == 53) +.weak sinl +.equ sinl, sin +#else +ENTRY(sinl) +END(sinl) +#endif + .section .rodata.npio2_hw,"a",%progbits .align 2 .Lnpio2_hw = . + 0 From b9f21a08f3590848cdda581c02b8b35808f6b3f8 Mon Sep 17 00:00:00 2001 From: Dmitriy Ivanov Date: Mon, 3 Nov 2014 21:13:55 -0800 Subject: [PATCH 16/89] Revert "Fix arm64 and arm builds." This reverts commit 445111a1c977e94a4233efd54f3690defa4a7582. Bug: 18222321 Bug: 18211780 Change-Id: I4fa9e1b63ec9b528f8bfed73c2ec15046c43a2fe --- tests/Android.mk | 14 ++++---------- tests/dlfcn_test.cpp | 4 ++-- tests/libs/Android.mk | 2 +- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/tests/Android.mk b/tests/Android.mk index 5a1127b96..6423df183 100644 --- a/tests/Android.mk +++ b/tests/Android.mk @@ -248,11 +248,8 @@ bionic-unit-tests_shared_libraries_target := \ libdl \ libpagemap \ libdl_preempt_test_1 \ - libdl_preempt_test_2 - -ifneq ($(filter $(TARGET_ARCH),arm arm64),$(TARGET_ARCH)) -bionic-unit-tests_shared_libraries_target += libdl_test_df_1_global -endif + libdl_preempt_test_2 \ + libdl_test_df_1_global module := bionic-unit-tests module_tag := optional @@ -296,11 +293,8 @@ bionic-unit-tests-glibc_src_files := \ bionic-unit-tests-glibc_shared_libraries := \ libdl_preempt_test_1 \ - libdl_preempt_test_2 - -ifneq ($(filter $(TARGET_ARCH),arm arm64),$(TARGET_ARCH)) -bionic-unit-tests-glibc_shared_libraries += libdl_test_df_1_global -endif + libdl_preempt_test_2 \ + libdl_test_df_1_global bionic-unit-tests-glibc_whole_static_libraries := \ libBionicStandardTests \ diff --git a/tests/dlfcn_test.cpp b/tests/dlfcn_test.cpp index 2f0d430f6..a7fe2f852 100644 --- a/tests/dlfcn_test.cpp +++ b/tests/dlfcn_test.cpp @@ -500,7 +500,7 @@ TEST(dlfcn, dlopen_nodelete_dt_flags_1) { } TEST(dlfcn, dlsym_df_1_global) { -#if !defined(__arm__) && !defined(__aarch64__) +#if !defined(__arm__) void* handle = dlopen("libtest_dlsym_df_1_global.so", RTLD_NOW); ASSERT_TRUE(handle != nullptr) << dlerror(); int (*get_answer)(); @@ -509,7 +509,7 @@ TEST(dlfcn, dlsym_df_1_global) { ASSERT_EQ(42, get_answer()); ASSERT_EQ(0, dlclose(handle)); #else - GTEST_LOG_(INFO) << "This test does nothing on arm/arm64 (to be reenabled once b/18137520 or b/18130452 are fixed).\n"; + GTEST_LOG_(INFO) << "This test does nothing on arm (to be reenabled once b/18137520 or b/18130452 are fixed).\n"; #endif } diff --git a/tests/libs/Android.mk b/tests/libs/Android.mk index 7d959d484..f45e5a80f 100644 --- a/tests/libs/Android.mk +++ b/tests/libs/Android.mk @@ -311,7 +311,7 @@ include $(LOCAL_PATH)/Android.build.testlib.mk # Library with DF_1_GLOBAL # ----------------------------------------------------------------------------- # TODO: re-enable arm once b/18137520 or b/18130452 are fixed -ifeq ($(filter $(TARGET_ARCH),arm arm64),) +ifeq ($(filter $(TARGET_ARCH),arm),) libdl_test_df_1_global_src_files := dl_df_1_global.cpp libdl_test_df_1_global_ldflags := -fuse-ld=bfd -Wl,-z,global module := libdl_test_df_1_global From 86fdf8f09bcc64a4f2033cd6463c8fc1a1ec70a1 Mon Sep 17 00:00:00 2001 From: Dmitriy Ivanov Date: Mon, 3 Nov 2014 21:14:07 -0800 Subject: [PATCH 17/89] Revert "Fix symbol lookup order during relocation" This reverts commit 976402cca13a1f4f3aa988fd301575e134ef5f2c. Bug: 18222321 Bug: 18211780 Change-Id: Iafdd3d843db7b1cf288be9a0232022816622c944 --- linker/linked_list.h | 12 --- linker/linker.cpp | 162 +++++++++++----------------- linker/linker.h | 23 ++-- tests/Android.mk | 13 +-- tests/dl_test.cpp | 72 ------------- tests/dlfcn_test.cpp | 14 --- tests/libs/Android.mk | 36 ------- tests/libs/dl_df_1_global.cpp | 19 ---- tests/libs/dl_df_1_use_global.cpp | 23 ---- tests/libs/dl_preempt_library_1.cpp | 45 -------- tests/libs/dl_preempt_library_2.cpp | 37 ------- 11 files changed, 71 insertions(+), 385 deletions(-) delete mode 100644 tests/dl_test.cpp delete mode 100644 tests/libs/dl_df_1_global.cpp delete mode 100644 tests/libs/dl_df_1_use_global.cpp delete mode 100644 tests/libs/dl_preempt_library_1.cpp delete mode 100644 tests/libs/dl_preempt_library_2.cpp diff --git a/linker/linked_list.h b/linker/linked_list.h index b08806161..72a32b4ba 100644 --- a/linker/linked_list.h +++ b/linker/linked_list.h @@ -36,12 +36,6 @@ class LinkedList { clear(); } - LinkedList(LinkedList&& that) { - this->head_ = that.head_; - this->tail_ = that.tail_; - that.head_ = that.tail_ = nullptr; - } - void push_front(T* const element) { LinkedListEntry* new_entry = Allocator::alloc(); new_entry->next = head_; @@ -146,12 +140,6 @@ class LinkedList { return false; } - static LinkedList make_list(T* const element) { - LinkedList one_element_list; - one_element_list.push_back(element); - return one_element_list; - } - private: LinkedListEntry* head_; LinkedListEntry* tail_; diff --git a/linker/linker.cpp b/linker/linker.cpp index ef6e3e127..f8963b59c 100644 --- a/linker/linker.cpp +++ b/linker/linker.cpp @@ -282,7 +282,7 @@ static void protect_data(int protection) { g_soinfo_links_allocator.protect_all(protection); } -static soinfo* soinfo_alloc(const char* name, struct stat* file_stat, off64_t file_offset, uint32_t rtld_flags) { +static soinfo* soinfo_alloc(const char* name, struct stat* file_stat, off64_t file_offset, int rtld_flags) { if (strlen(name) >= SOINFO_NAME_LEN) { DL_ERR("library name \"%s\" too long", name); return nullptr; @@ -481,8 +481,7 @@ static unsigned elfhash(const char* _name) { return h; } -static ElfW(Sym)* soinfo_do_lookup(soinfo* si_from, const char* name, soinfo** si_found_in, - const soinfo::soinfo_list_t& global_group, const soinfo::soinfo_list_t& local_group) { +static ElfW(Sym)* soinfo_do_lookup(soinfo* si, const char* name, soinfo** lsi, const soinfo::soinfo_list_t& local_group) { unsigned elf_hash = elfhash(name); ElfW(Sym)* s = nullptr; @@ -497,40 +496,49 @@ static ElfW(Sym)* soinfo_do_lookup(soinfo* si_from, const char* name, soinfo** s * Note that this is unlikely since static linker avoids generating * relocations for -Bsymbolic linked dynamic executables. */ - if (si_from->has_DT_SYMBOLIC) { - DEBUG("%s: looking up %s in local scope (DT_SYMBOLIC)", si_from->name, name); - s = soinfo_elf_lookup(si_from, elf_hash, name); + if (si->has_DT_SYMBOLIC) { + DEBUG("%s: looking up %s in local scope (DT_SYMBOLIC)", si->name, name); + s = soinfo_elf_lookup(si, elf_hash, name); if (s != nullptr) { - *si_found_in = si_from; + *lsi = si; } } - // 1. Look for it in global_group - if (s == nullptr) { - global_group.visit([&](soinfo* global_si) { - DEBUG("%s: looking up %s in %s (from global group)", si_from->name, name, global_si->name); - s = soinfo_elf_lookup(global_si, elf_hash, name); + if (s == nullptr && somain != nullptr) { + // 1. Look for it in the main executable unless we already did. + if (si != somain || !si->has_DT_SYMBOLIC) { + DEBUG("%s: looking up %s in executable %s", + si->name, name, somain->name); + s = soinfo_elf_lookup(somain, elf_hash, name); if (s != nullptr) { - *si_found_in = global_si; - return false; + *lsi = somain; } + } - return true; - }); + // 2. Look for it in the ld_preloads + if (s == nullptr) { + for (int i = 0; g_ld_preloads[i] != NULL; i++) { + s = soinfo_elf_lookup(g_ld_preloads[i], elf_hash, name); + if (s != nullptr) { + *lsi = g_ld_preloads[i]; + break; + } + } + } } - // 2. Look for it in the local group + // 3. Look for it in the local group if (s == nullptr) { local_group.visit([&](soinfo* local_si) { - if (local_si == si_from && si_from->has_DT_SYMBOLIC) { + if (local_si == si && si->has_DT_SYMBOLIC) { // we already did this - skip return true; } - DEBUG("%s: looking up %s in %s (from local group)", si_from->name, name, local_si->name); + DEBUG("%s: looking up %s in %s (from local group)", si->name, name, local_si->name); s = soinfo_elf_lookup(local_si, elf_hash, name); if (s != nullptr) { - *si_found_in = local_si; + *lsi = local_si; return false; } @@ -541,9 +549,9 @@ static ElfW(Sym)* soinfo_do_lookup(soinfo* si_from, const char* name, soinfo** s if (s != nullptr) { TRACE_TYPE(LOOKUP, "si %s sym %s s->st_value = %p, " "found in %s, base = %p, load bias = %p", - si_from->name, name, reinterpret_cast(s->st_value), - (*si_found_in)->name, reinterpret_cast((*si_found_in)->base), - reinterpret_cast((*si_found_in)->load_bias)); + si->name, name, reinterpret_cast(s->st_value), + (*lsi)->name, reinterpret_cast((*lsi)->base), + reinterpret_cast((*lsi)->load_bias)); } return s; @@ -908,24 +916,6 @@ static bool is_recursive(soinfo* si, soinfo* parent) { }); } -// TODO: this is slightly unusual way to construct -// the global group for relocation. Not every RTLD_GLOBAL -// library is included in this group for backwards-compatibility -// reasons. -// -// This group consists of the main executable, LD_PRELOADs -// and libraries with the DF_1_GLOBAL flag set. -static soinfo::soinfo_list_t make_global_group() { - soinfo::soinfo_list_t global_group; - for (soinfo* si = somain; si != nullptr; si = si->next) { - if ((si->get_dt_flags_1() & DF_1_GLOBAL) != 0) { - global_group.push_back(si); - } - } - - return global_group; -} - static bool find_libraries(soinfo* start_with, const char* const library_names[], size_t library_names_count, soinfo* soinfos[], soinfo* ld_preloads[], size_t ld_preloads_count, int rtld_flags, const android_dlextinfo* extinfo) { // Step 0: prepare. @@ -935,9 +925,6 @@ static bool find_libraries(soinfo* start_with, const char* const library_names[] load_tasks.push_back(LoadTask::create(name, start_with)); } - // Construct global_group. - soinfo::soinfo_list_t global_group = make_global_group(); - // If soinfos array is null allocate one on stack. // The array is needed in case of failure; for example // when library_names[] = {libone.so, libtwo.so} and libone.so @@ -986,11 +973,6 @@ static bool find_libraries(soinfo* start_with, const char* const library_names[] // When ld_preloads is not null, the first // ld_preloads_count libs are in fact ld_preloads. if (ld_preloads != nullptr && soinfos_count < ld_preloads_count) { - // Add LD_PRELOADed libraries to the global group for future runs. - // There is no need to explicitly add them to the global group - // for this run because they are going to appear in the local - // group in the correct order. - si->set_dt_flags_1(si->get_dt_flags_1() | DF_1_GLOBAL); ld_preloads[soinfos_count] = si; } @@ -1011,7 +993,7 @@ static bool find_libraries(soinfo* start_with, const char* const library_names[] bool linked = local_group.visit([&](soinfo* si) { if ((si->flags & FLAG_LINKED) == 0) { - if (!si->LinkImage(global_group, local_group, extinfo)) { + if (!si->LinkImage(local_group, extinfo)) { return false; } si->flags |= FLAG_LINKED; @@ -1146,7 +1128,7 @@ static ElfW(Addr) call_ifunc_resolver(ElfW(Addr) resolver_addr) { } #if defined(USE_RELA) -int soinfo::Relocate(ElfW(Rela)* rela, unsigned count, const soinfo_list_t& global_group, const soinfo_list_t& local_group) { +int soinfo::Relocate(ElfW(Rela)* rela, unsigned count, const soinfo_list_t& local_group) { for (size_t idx = 0; idx < count; ++idx, ++rela) { unsigned type = ELFW(R_TYPE)(rela->r_info); unsigned sym = ELFW(R_SYM)(rela->r_info); @@ -1164,7 +1146,7 @@ int soinfo::Relocate(ElfW(Rela)* rela, unsigned count, const soinfo_list_t& glob if (sym != 0) { sym_name = get_string(symtab[sym].st_name); - s = soinfo_do_lookup(this, sym_name, &lsi, global_group,local_group); + s = soinfo_do_lookup(this, sym_name, &lsi, local_group); if (s == nullptr) { // We only allow an undefined symbol if this is a weak reference... s = &symtab[sym]; @@ -1423,7 +1405,7 @@ int soinfo::Relocate(ElfW(Rela)* rela, unsigned count, const soinfo_list_t& glob } #else // REL, not RELA. -int soinfo::Relocate(ElfW(Rel)* rel, unsigned count, const soinfo_list_t& global_group, const soinfo_list_t& local_group) { +int soinfo::Relocate(ElfW(Rel)* rel, unsigned count, const soinfo_list_t& local_group) { for (size_t idx = 0; idx < count; ++idx, ++rel) { unsigned type = ELFW(R_TYPE)(rel->r_info); // TODO: don't use unsigned for 'sym'. Use uint32_t or ElfW(Addr) instead. @@ -1442,7 +1424,7 @@ int soinfo::Relocate(ElfW(Rel)* rel, unsigned count, const soinfo_list_t& global if (sym != 0) { sym_name = get_string(symtab[sym].st_name); - s = soinfo_do_lookup(this, sym_name, &lsi, global_group, local_group); + s = soinfo_do_lookup(this, sym_name, &lsi, local_group); if (s == nullptr) { // We only allow an undefined symbol if this is a weak reference... s = &symtab[sym]; @@ -1628,7 +1610,7 @@ int soinfo::Relocate(ElfW(Rel)* rel, unsigned count, const soinfo_list_t& global #endif #if defined(__mips__) -static bool mips_relocate_got(soinfo* si, const soinfo::soinfo_list_t& global_group, const soinfo::soinfo_list_t& local_group) { +static bool mips_relocate_got(soinfo* si, const soinfo::soinfo_list_t& local_group) { ElfW(Addr)** got = si->plt_got; if (got == nullptr) { return true; @@ -1661,7 +1643,7 @@ static bool mips_relocate_got(soinfo* si, const soinfo::soinfo_list_t& global_gr // This is an undefined reference... try to locate it. const char* sym_name = si->get_string(sym->st_name); soinfo* lsi = nullptr; - ElfW(Sym)* s = soinfo_do_lookup(si, sym_name, &lsi, global_group, local_group); + ElfW(Sym)* s = soinfo_do_lookup(si, sym_name, &lsi, local_group); if (s == nullptr) { // We only allow an undefined symbol if this is a weak reference. s = &symtab[g]; @@ -1801,7 +1783,7 @@ void soinfo::remove_all_links() { children.clear(); } -dev_t soinfo::get_st_dev() const { +dev_t soinfo::get_st_dev() { if (has_min_version(0)) { return st_dev; } @@ -1809,7 +1791,7 @@ dev_t soinfo::get_st_dev() const { return 0; }; -ino_t soinfo::get_st_ino() const { +ino_t soinfo::get_st_ino() { if (has_min_version(0)) { return st_ino; } @@ -1817,7 +1799,7 @@ ino_t soinfo::get_st_ino() const { return 0; } -off64_t soinfo::get_file_offset() const { +off64_t soinfo::get_file_offset() { if (has_min_version(1)) { return file_offset; } @@ -1825,7 +1807,7 @@ off64_t soinfo::get_file_offset() const { return 0; } -uint32_t soinfo::get_rtld_flags() const { +int soinfo::get_rtld_flags() { if (has_min_version(1)) { return rtld_flags; } @@ -1833,27 +1815,6 @@ uint32_t soinfo::get_rtld_flags() const { return 0; } -uint32_t soinfo::get_dt_flags_1() const { - if (has_min_version(1)) { - return dt_flags_1; - } - - return 0; -} -void soinfo::set_dt_flags_1(uint32_t dt_flags_1) { - if (has_min_version(1)) { - if ((dt_flags_1 & DF_1_GLOBAL) != 0) { - rtld_flags |= RTLD_GLOBAL; - } - - if ((dt_flags_1 & DF_1_NODELETE) != 0) { - rtld_flags |= RTLD_NODELETE; - } - - this->dt_flags_1 = dt_flags_1; - } -} - // This is a return on get_children()/get_parents() if // 'this->flags' does not have FLAG_NEW_SOINFO set. static soinfo::soinfo_list_t g_empty_list; @@ -1891,9 +1852,8 @@ const char* soinfo::get_string(ElfW(Word) index) const { } bool soinfo::can_unload() const { - return (get_rtld_flags() & (RTLD_NODELETE | RTLD_GLOBAL)) == 0; + return (rtld_flags & (RTLD_NODELETE | RTLD_GLOBAL)) == 0; } - /* Force any of the closed stdin, stdout and stderr to be associated with /dev/null. */ static int nullify_closed_stdio() { @@ -2194,9 +2154,16 @@ bool soinfo::PrelinkImage() { break; case DT_FLAGS_1: - set_dt_flags_1(d->d_un.d_val); + if ((d->d_un.d_val & DF_1_GLOBAL) != 0) { + rtld_flags |= RTLD_GLOBAL; + } + + if ((d->d_un.d_val & DF_1_NODELETE) != 0) { + rtld_flags |= RTLD_NODELETE; + } + // TODO: Implement other flags - if ((d->d_un.d_val & ~SUPPORTED_DT_FLAGS_1) != 0) { + if ((d->d_un.d_val & ~(DF_1_NOW | DF_1_GLOBAL | DF_1_NODELETE)) != 0) { DL_WARN("Unsupported flags DT_FLAGS_1=%p", reinterpret_cast(d->d_un.d_val)); } break; @@ -2269,7 +2236,7 @@ bool soinfo::PrelinkImage() { return true; } -bool soinfo::LinkImage(const soinfo_list_t& global_group, const soinfo_list_t& local_group, const android_dlextinfo* extinfo) { +bool soinfo::LinkImage(const soinfo_list_t& local_group, const android_dlextinfo* extinfo) { #if !defined(__LP64__) if (has_text_relocations) { @@ -2288,33 +2255,33 @@ bool soinfo::LinkImage(const soinfo_list_t& global_group, const soinfo_list_t& l #if defined(USE_RELA) if (rela != nullptr) { DEBUG("[ relocating %s ]", name); - if (Relocate(rela, rela_count, global_group, local_group)) { + if (Relocate(rela, rela_count, local_group)) { return false; } } if (plt_rela != nullptr) { DEBUG("[ relocating %s plt ]", name); - if (Relocate(plt_rela, plt_rela_count, global_group, local_group)) { + if (Relocate(plt_rela, plt_rela_count, local_group)) { return false; } } #else if (rel != nullptr) { DEBUG("[ relocating %s ]", name); - if (Relocate(rel, rel_count, global_group, local_group)) { + if (Relocate(rel, rel_count, local_group)) { return false; } } if (plt_rel != nullptr) { DEBUG("[ relocating %s plt ]", name); - if (Relocate(plt_rel, plt_rel_count, global_group, local_group)) { + if (Relocate(plt_rel, plt_rel_count, local_group)) { return false; } } #endif #if defined(__mips__) - if (!mips_relocate_got(this, global_group, local_group)) { + if (!mips_relocate_got(this, local_group)) { return false; } #endif @@ -2381,7 +2348,7 @@ static void add_vdso(KernelArgumentBlock& args __unused) { si->load_bias = get_elf_exec_load_bias(ehdr_vdso); si->PrelinkImage(); - si->LinkImage(g_empty_list, soinfo::soinfo_list_t::make_list(si), nullptr); + si->LinkImage(g_empty_list, nullptr); #endif } @@ -2512,9 +2479,6 @@ static ElfW(Addr) __linker_init_post_relocation(KernelArgumentBlock& args, ElfW( si->PrelinkImage(); - // add somain to global group - si->set_dt_flags_1(si->get_dt_flags_1() | DF_1_GLOBAL); - // Load ld_preloads and dependencies. StringLinkedList needed_library_name_list; size_t needed_libraries_count = 0; @@ -2658,13 +2622,7 @@ extern "C" ElfW(Addr) __linker_init(void* raw_args) { linker_so.phnum = elf_hdr->e_phnum; linker_so.flags |= FLAG_LINKER; - // This might not be obvious... The reasons why we pass g_empty_list - // in place of local_group here are (1) we do not really need it, because - // linker is built with DT_SYMBOLIC and therefore relocates its symbols against - // itself without having to look into local_group and (2) allocators - // are not yet initialized, and therefore we cannot use linked_list.push_* - // functions at this point. - if (!(linker_so.PrelinkImage() && linker_so.LinkImage(g_empty_list, g_empty_list, nullptr))) { + if (!(linker_so.PrelinkImage() && linker_so.LinkImage(g_empty_list, nullptr))) { // It would be nice to print an error message, but if the linker // can't link itself, there's no guarantee that we'll be able to // call write() (because it involves a GOT reference). We may as diff --git a/linker/linker.h b/linker/linker.h index 0a98b40dc..222aca11e 100644 --- a/linker/linker.h +++ b/linker/linker.h @@ -89,9 +89,7 @@ #define FLAG_LINKER 0x00000010 // The linker itself #define FLAG_NEW_SOINFO 0x40000000 // new soinfo format -#define SUPPORTED_DT_FLAGS_1 (DF_1_NOW | DF_1_GLOBAL | DF_1_NODELETE) - -#define SOINFO_VERSION 1 +#define SOINFO_VERSION 0 #define SOINFO_NAME_LEN 128 @@ -209,18 +207,16 @@ struct soinfo { void CallDestructors(); void CallPreInitConstructors(); bool PrelinkImage(); - bool LinkImage(const soinfo_list_t& global_group, const soinfo_list_t& local_group, const android_dlextinfo* extinfo); + bool LinkImage(const soinfo_list_t& local_group, const android_dlextinfo* extinfo); void add_child(soinfo* child); void remove_all_links(); - ino_t get_st_ino() const; - dev_t get_st_dev() const; - off64_t get_file_offset() const; + ino_t get_st_ino(); + dev_t get_st_dev(); + off64_t get_file_offset(); - uint32_t get_rtld_flags() const; - uint32_t get_dt_flags_1() const; - void set_dt_flags_1(uint32_t dt_flags_1); + int get_rtld_flags(); soinfo_list_t& get_children(); soinfo_list_t& get_parents(); @@ -238,9 +234,9 @@ struct soinfo { void CallArray(const char* array_name, linker_function_t* functions, size_t count, bool reverse); void CallFunction(const char* function_name, linker_function_t function); #if defined(USE_RELA) - int Relocate(ElfW(Rela)* rela, unsigned count, const soinfo_list_t& global_group, const soinfo_list_t& local_group); + int Relocate(ElfW(Rela)* rela, unsigned count, const soinfo_list_t& local_group); #else - int Relocate(ElfW(Rel)* rel, unsigned count, const soinfo_list_t& global_group, const soinfo_list_t& local_group); + int Relocate(ElfW(Rel)* rel, unsigned count, const soinfo_list_t& local_group); #endif private: @@ -258,8 +254,7 @@ struct soinfo { // version >= 1 off64_t file_offset; - uint32_t rtld_flags; - uint32_t dt_flags_1; + int rtld_flags; size_t strtab_size; friend soinfo* get_libdl_info(); diff --git a/tests/Android.mk b/tests/Android.mk index 6423df183..8b0b0a0e0 100644 --- a/tests/Android.mk +++ b/tests/Android.mk @@ -225,7 +225,6 @@ bionic-unit-tests_whole_static_libraries := \ bionic-unit-tests_src_files := \ atexit_test.cpp \ - dl_test.cpp \ dlext_test.cpp \ dlfcn_test.cpp \ @@ -238,7 +237,8 @@ bionic-unit-tests_conlyflags := \ bionic-unit-tests_cppflags := $(test_cppflags) bionic-unit-tests_ldflags := \ - -Wl,--export-dynamic + -Wl,--export-dynamic \ + -Wl,-u,DlSymTestFunction \ bionic-unit-tests_c_includes := \ bionic/libc \ @@ -247,9 +247,6 @@ bionic-unit-tests_c_includes := \ bionic-unit-tests_shared_libraries_target := \ libdl \ libpagemap \ - libdl_preempt_test_1 \ - libdl_preempt_test_2 \ - libdl_test_df_1_global module := bionic-unit-tests module_tag := optional @@ -289,12 +286,6 @@ ifeq ($(HOST_OS)-$(HOST_ARCH),$(filter $(HOST_OS)-$(HOST_ARCH),linux-x86 linux-x bionic-unit-tests-glibc_src_files := \ atexit_test.cpp \ dlfcn_test.cpp \ - dl_test.cpp \ - -bionic-unit-tests-glibc_shared_libraries := \ - libdl_preempt_test_1 \ - libdl_preempt_test_2 \ - libdl_test_df_1_global bionic-unit-tests-glibc_whole_static_libraries := \ libBionicStandardTests \ diff --git a/tests/dl_test.cpp b/tests/dl_test.cpp deleted file mode 100644 index 74c7b510f..000000000 --- a/tests/dl_test.cpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (C) 2012 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include - -#include - -extern "C" int main_global_default_serial() { - return 3370318; -} - -extern "C" int main_global_protected_serial() { - return 2716057; -} - -// The following functions are defined in DT_NEEDED -// libdl_preempt_test.so library. - -// This one calls main_global_default_serial -extern "C" int main_global_default_get_serial(); - -// This one calls main_global_protected_serial -extern "C" int main_global_protected_get_serial(); - -// This one calls lib_global_default_serial -extern "C" int lib_global_default_get_serial(); - -// This one calls lib_global_protected_serial -extern "C" int lib_global_protected_get_serial(); - -// This test verifies that the global default function -// main_global_default_serial() is preempted by -// the function defined above. -TEST(dl, main_preempts_global_default) { - ASSERT_EQ(3370318, main_global_default_get_serial()); -} - -// This one makes sure that the global protected -// symbols do not get preempted -TEST(dl, main_does_not_preempt_global_protected) { - ASSERT_EQ(3370318, main_global_protected_get_serial()); -} - -// check same things for lib -TEST(dl, lib_preempts_global_default) { - ASSERT_EQ(3370318, lib_global_default_get_serial()); -} - -TEST(dl, lib_does_not_preempt_global_protected) { - ASSERT_EQ(3370318, lib_global_protected_get_serial()); -} - -// TODO: Add tests for LD_PRELOADs diff --git a/tests/dlfcn_test.cpp b/tests/dlfcn_test.cpp index a7fe2f852..e604f5abe 100644 --- a/tests/dlfcn_test.cpp +++ b/tests/dlfcn_test.cpp @@ -499,20 +499,6 @@ TEST(dlfcn, dlopen_nodelete_dt_flags_1) { ASSERT_TRUE(!is_unloaded); } -TEST(dlfcn, dlsym_df_1_global) { -#if !defined(__arm__) - void* handle = dlopen("libtest_dlsym_df_1_global.so", RTLD_NOW); - ASSERT_TRUE(handle != nullptr) << dlerror(); - int (*get_answer)(); - get_answer = reinterpret_cast(dlsym(handle, "dl_df_1_global_get_answer")); - ASSERT_TRUE(get_answer != nullptr) << dlerror(); - ASSERT_EQ(42, get_answer()); - ASSERT_EQ(0, dlclose(handle)); -#else - GTEST_LOG_(INFO) << "This test does nothing on arm (to be reenabled once b/18137520 or b/18130452 are fixed).\n"; -#endif -} - TEST(dlfcn, dlopen_failure) { void* self = dlopen("/does/not/exist", RTLD_NOW); ASSERT_TRUE(self == NULL); diff --git a/tests/libs/Android.mk b/tests/libs/Android.mk index f45e5a80f..05e7113ba 100644 --- a/tests/libs/Android.mk +++ b/tests/libs/Android.mk @@ -289,42 +289,6 @@ libtest_atexit_src_files := \ module := libtest_atexit include $(LOCAL_PATH)/Android.build.testlib.mk -# ----------------------------------------------------------------------------- -# This library is used by dl_load test to check symbol preempting -# by main executable -# ----------------------------------------------------------------------------- -libdl_preempt_test_1_src_files := dl_preempt_library_1.cpp - -module := libdl_preempt_test_1 -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# This library is used by dl_load test to check symbol preempting -# by libdl_preempt_test_1.so -# ----------------------------------------------------------------------------- -libdl_preempt_test_2_src_files := dl_preempt_library_2.cpp - -module := libdl_preempt_test_2 -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# Library with DF_1_GLOBAL -# ----------------------------------------------------------------------------- -# TODO: re-enable arm once b/18137520 or b/18130452 are fixed -ifeq ($(filter $(TARGET_ARCH),arm),) -libdl_test_df_1_global_src_files := dl_df_1_global.cpp -libdl_test_df_1_global_ldflags := -fuse-ld=bfd -Wl,-z,global -module := libdl_test_df_1_global -include $(LOCAL_PATH)/Android.build.testlib.mk -endif - -# ----------------------------------------------------------------------------- -# Library using symbol from libdl_test_df_1_global -# ----------------------------------------------------------------------------- -libtest_dlsym_df_1_global_src_files := dl_df_1_use_global.cpp -module := libtest_dlsym_df_1_global -include $(LOCAL_PATH)/Android.build.testlib.mk - # ----------------------------------------------------------------------------- # Library with weak function # ----------------------------------------------------------------------------- diff --git a/tests/libs/dl_df_1_global.cpp b/tests/libs/dl_df_1_global.cpp deleted file mode 100644 index 39856fd50..000000000 --- a/tests/libs/dl_df_1_global.cpp +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -extern "C" int dl_df_1_global_get_answer_impl() { - return 42; -} diff --git a/tests/libs/dl_df_1_use_global.cpp b/tests/libs/dl_df_1_use_global.cpp deleted file mode 100644 index e14910d1c..000000000 --- a/tests/libs/dl_df_1_use_global.cpp +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -extern "C" int __attribute__((weak)) dl_df_1_global_get_answer_impl() { - return 0; -} - -extern "C" int dl_df_1_global_get_answer() { - return dl_df_1_global_get_answer_impl(); -} diff --git a/tests/libs/dl_preempt_library_1.cpp b/tests/libs/dl_preempt_library_1.cpp deleted file mode 100644 index b4d81d5ac..000000000 --- a/tests/libs/dl_preempt_library_1.cpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// This one should be preempted by the function -// defined in the main executable. -extern "C" int __attribute__((weak)) main_global_default_serial() { - return 2716057; -} - -// Even though this one is defined by the main -// executable it should not be preempted -// because of protected visibility -extern "C" int __attribute__((weak, visibility("protected"))) main_global_protected_serial() { - return 3370318; -} - -extern "C" int main_global_default_get_serial() { - return main_global_default_serial(); -} - -extern "C" int main_global_protected_get_serial() { - return main_global_protected_serial(); -} - -// Trying to preempt functions from a DT_NEEDED .so -extern "C" int lib_global_default_serial() { - return 3370318; -} - -extern "C" int lib_global_protected_serial() { - return 2716057; -} diff --git a/tests/libs/dl_preempt_library_2.cpp b/tests/libs/dl_preempt_library_2.cpp deleted file mode 100644 index 8df9a1667..000000000 --- a/tests/libs/dl_preempt_library_2.cpp +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// This one should be preempted by the function -// defined in libdl_preempt_test_1.so -extern "C" int __attribute__((weak)) lib_global_default_serial() { - return 2716057; -} - -// Even though this one is defined by -// libdl_preempt_test_1.so it should not be -// preempted because of protected visibility -extern "C" int __attribute__((weak,visibility("protected"))) lib_global_protected_serial() { - return 3370318; -} - -extern "C" int lib_global_default_get_serial() { - return lib_global_default_serial(); -} - -extern "C" int lib_global_protected_get_serial() { - return lib_global_protected_serial(); -} - From 8b952f55527a988d63ea277bf560c0527993f8d5 Mon Sep 17 00:00:00 2001 From: Dmitriy Ivanov Date: Mon, 3 Nov 2014 21:14:31 -0800 Subject: [PATCH 18/89] Revert "Fix mips build" This reverts commit bf3d5ef5fd240d4c5fbde1b32f9084dbc720840b. Bug: 18222321 Bug: 18211780 Change-Id: I902ed888197b358c77303f1acb6d5ffd7ae6dcd3 --- linker/linker.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linker/linker.cpp b/linker/linker.cpp index f8963b59c..c63ac35bb 100644 --- a/linker/linker.cpp +++ b/linker/linker.cpp @@ -2281,7 +2281,7 @@ bool soinfo::LinkImage(const soinfo_list_t& local_group, const android_dlextinfo #endif #if defined(__mips__) - if (!mips_relocate_got(this, local_group)) { + if (!mips_relocate_got(this)) { return false; } #endif From 8bf7353b79c3249951831378a6605fd9e8a7def1 Mon Sep 17 00:00:00 2001 From: Dmitriy Ivanov Date: Mon, 3 Nov 2014 21:15:15 -0800 Subject: [PATCH 19/89] Revert "Remove unnecessary lookups during relocations" This reverts commit 6442dbd3bcadbd5e522465743a8d8cf56338ae1c. Bug: 18222321 Bug: 18211780 Change-Id: I87b18a32238a1f75afe56149221b6691f50d9f56 --- linker/linker.cpp | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/linker/linker.cpp b/linker/linker.cpp index c63ac35bb..eb1a483ae 100644 --- a/linker/linker.cpp +++ b/linker/linker.cpp @@ -530,11 +530,6 @@ static ElfW(Sym)* soinfo_do_lookup(soinfo* si, const char* name, soinfo** lsi, c // 3. Look for it in the local group if (s == nullptr) { local_group.visit([&](soinfo* local_si) { - if (local_si == si && si->has_DT_SYMBOLIC) { - // we already did this - skip - return true; - } - DEBUG("%s: looking up %s in %s (from local group)", si->name, name, local_si->name); s = soinfo_elf_lookup(local_si, elf_hash, name); if (s != nullptr) { @@ -546,6 +541,28 @@ static ElfW(Sym)* soinfo_do_lookup(soinfo* si, const char* name, soinfo** lsi, c }); } + // 4. Look for it in this library (unless we already did it because of DT_SYMBOLIC) + if (s == nullptr && !si->has_DT_SYMBOLIC) { + DEBUG("%s: looking up %s in local scope", si->name, name); + s = soinfo_elf_lookup(si, elf_hash, name); + if (s != nullptr) { + *lsi = si; + } + } + + // 5. Dependencies + if (s == nullptr) { + si->get_children().visit([&](soinfo* child) { + DEBUG("%s: looking up %s in %s", si->name, name, child->name); + s = soinfo_elf_lookup(child, elf_hash, name); + if (s != nullptr) { + *lsi = child; + return false; + } + return true; + }); + } + if (s != nullptr) { TRACE_TYPE(LOOKUP, "si %s sym %s s->st_value = %p, " "found in %s, base = %p, load bias = %p", From 189ac9f14280d495d3d6f3c196f895e7b8e6098f Mon Sep 17 00:00:00 2001 From: Dmitriy Ivanov Date: Mon, 3 Nov 2014 21:15:25 -0800 Subject: [PATCH 20/89] Revert "Fix relocation to look for symbols in local group" This reverts commit fd2747bb585fc51b5ad56db09c0e9b66c7091a92. Bug: 18222321 Bug: 18211780 Change-Id: I2d4ebab1e73b7277161af76b99f8249825b22d65 --- linker/linked_list.h | 4 +- linker/linker.cpp | 193 +++++++----------- linker/linker.h | 6 +- tests/dlfcn_test.cpp | 186 ++--------------- .../Android.build.dlopen_check_order_dlsym.mk | 90 -------- ...lopen_check_order_reloc_main_executable.mk | 56 ----- ...build.dlopen_check_order_reloc_siblings.mk | 133 ------------ tests/libs/Android.mk | 75 ++++++- .../libs/dlopen_check_order_reloc_answer.cpp | 23 --- .../dlopen_check_order_reloc_answer_impl.cpp | 19 -- ...dlopen_check_order_reloc_nephew_answer.cpp | 21 -- .../dlopen_check_order_reloc_root_answer.cpp | 21 -- ...pen_check_order_reloc_root_answer_impl.cpp | 19 -- ...m_answer.cpp => dlopen_testlib_answer.cpp} | 4 +- 14 files changed, 162 insertions(+), 688 deletions(-) delete mode 100644 tests/libs/Android.build.dlopen_check_order_dlsym.mk delete mode 100644 tests/libs/Android.build.dlopen_check_order_reloc_main_executable.mk delete mode 100644 tests/libs/Android.build.dlopen_check_order_reloc_siblings.mk delete mode 100644 tests/libs/dlopen_check_order_reloc_answer.cpp delete mode 100644 tests/libs/dlopen_check_order_reloc_answer_impl.cpp delete mode 100644 tests/libs/dlopen_check_order_reloc_nephew_answer.cpp delete mode 100644 tests/libs/dlopen_check_order_reloc_root_answer.cpp delete mode 100644 tests/libs/dlopen_check_order_reloc_root_answer_impl.cpp rename tests/libs/{dlopen_check_order_dlsym_answer.cpp => dlopen_testlib_answer.cpp} (87%) diff --git a/linker/linked_list.h b/linker/linked_list.h index 72a32b4ba..4e62e208f 100644 --- a/linker/linked_list.h +++ b/linker/linked_list.h @@ -86,7 +86,7 @@ class LinkedList { } template - void for_each(F action) const { + void for_each(F action) { visit([&] (T* si) { action(si); return true; @@ -94,7 +94,7 @@ class LinkedList { } template - bool visit(F action) const { + bool visit(F action) { for (LinkedListEntry* e = head_; e != nullptr; e = e->next) { if (!action(e->element)) { return false; diff --git a/linker/linker.cpp b/linker/linker.cpp index eb1a483ae..41557e231 100644 --- a/linker/linker.cpp +++ b/linker/linker.cpp @@ -415,7 +415,7 @@ int dl_iterate_phdr(int (*cb)(dl_phdr_info* info, size_t size, void* data), void return rv; } -static ElfW(Sym)* soinfo_elf_lookup(const soinfo* si, unsigned hash, const char* name) { +static ElfW(Sym)* soinfo_elf_lookup(soinfo* si, unsigned hash, const char* name) { ElfW(Sym)* symtab = si->symtab; TRACE_TYPE(LOOKUP, "SEARCH %s in %s@%p %x %zd", @@ -481,7 +481,7 @@ static unsigned elfhash(const char* _name) { return h; } -static ElfW(Sym)* soinfo_do_lookup(soinfo* si, const char* name, soinfo** lsi, const soinfo::soinfo_list_t& local_group) { +static ElfW(Sym)* soinfo_do_lookup(soinfo* si, const char* name, soinfo** lsi) { unsigned elf_hash = elfhash(name); ElfW(Sym)* s = nullptr; @@ -527,21 +527,16 @@ static ElfW(Sym)* soinfo_do_lookup(soinfo* si, const char* name, soinfo** lsi, c } } - // 3. Look for it in the local group - if (s == nullptr) { - local_group.visit([&](soinfo* local_si) { - DEBUG("%s: looking up %s in %s (from local group)", si->name, name, local_si->name); - s = soinfo_elf_lookup(local_si, elf_hash, name); - if (s != nullptr) { - *lsi = local_si; - return false; - } - - return true; - }); - } + /* Look for symbols in the local scope (the object who is + * searching). This happens with C++ templates on x86 for some + * reason. + * + * Notes on weak symbols: + * The ELF specs are ambiguous about treatment of weak definitions in + * dynamic linking. Some systems return the first definition found + * and some the first non-weak definition. This is system dependent. + * Here we return the first definition found for simplicity. */ - // 4. Look for it in this library (unless we already did it because of DT_SYMBOLIC) if (s == nullptr && !si->has_DT_SYMBOLIC) { DEBUG("%s: looking up %s in local scope", si->name, name); s = soinfo_elf_lookup(si, elf_hash, name); @@ -550,7 +545,6 @@ static ElfW(Sym)* soinfo_do_lookup(soinfo* si, const char* name, soinfo** lsi, c } } - // 5. Dependencies if (s == nullptr) { si->get_children().visit([&](soinfo* child) { DEBUG("%s: looking up %s in %s", si->name, name, child->name); @@ -649,61 +643,33 @@ typedef linked_list_t StringLinkedList; typedef linked_list_t LoadTaskList; -// This function walks down the tree of soinfo dependencies -// in breadth-first order and -// * calls action(soinfo* si) for each node, and -// * terminates walk if action returns false. -// -// walk_dependencies_tree returns false if walk was terminated -// by the action and true otherwise. -template -static bool walk_dependencies_tree(soinfo* root_soinfos[], size_t root_soinfos_size, F action) { +// This is used by dlsym(3). It performs symbol lookup only within the +// specified soinfo object and its dependencies in breadth first order. +ElfW(Sym)* dlsym_handle_lookup(soinfo* si, soinfo** found, const char* name) { SoinfoLinkedList visit_list; SoinfoLinkedList visited; - for (size_t i = 0; i < root_soinfos_size; ++i) { - visit_list.push_back(root_soinfos[i]); - } - - soinfo* si; - while ((si = visit_list.pop_front()) != nullptr) { - if (visited.contains(si)) { + visit_list.push_back(si); + soinfo* current_soinfo; + while ((current_soinfo = visit_list.pop_front()) != nullptr) { + if (visited.contains(current_soinfo)) { continue; } - if (!action(si)) { - return false; - } - - visited.push_back(si); - - si->get_children().for_each([&](soinfo* child) { - visit_list.push_back(child); - }); - } - - return true; -} - + ElfW(Sym)* result = soinfo_elf_lookup(current_soinfo, elfhash(name), name); -// This is used by dlsym(3). It performs symbol lookup only within the -// specified soinfo object and its dependencies in breadth first order. -ElfW(Sym)* dlsym_handle_lookup(soinfo* si, soinfo** found, const char* name) { - ElfW(Sym)* result = nullptr; - uint32_t elf_hash = elfhash(name); - - - walk_dependencies_tree(&si, 1, [&](soinfo* current_soinfo) { - result = soinfo_elf_lookup(current_soinfo, elf_hash, name); if (result != nullptr) { *found = current_soinfo; - return false; + return result; } + visited.push_back(current_soinfo); - return true; - }); + current_soinfo->get_children().for_each([&](soinfo* child) { + visit_list.push_back(child); + }); + } - return result; + return nullptr; } /* This is used by dlsym(3) to performs a global symbol lookup. If the @@ -933,30 +899,19 @@ static bool is_recursive(soinfo* si, soinfo* parent) { }); } -static bool find_libraries(soinfo* start_with, const char* const library_names[], size_t library_names_count, soinfo* soinfos[], - soinfo* ld_preloads[], size_t ld_preloads_count, int rtld_flags, const android_dlextinfo* extinfo) { +static bool find_libraries(const char* const library_names[], size_t library_names_size, soinfo* soinfos[], + soinfo* ld_preloads[], size_t ld_preloads_size, int rtld_flags, const android_dlextinfo* extinfo) { // Step 0: prepare. LoadTaskList load_tasks; - for (size_t i = 0; i < library_names_count; ++i) { + for (size_t i = 0; i < library_names_size; ++i) { const char* name = library_names[i]; - load_tasks.push_back(LoadTask::create(name, start_with)); - } - - // If soinfos array is null allocate one on stack. - // The array is needed in case of failure; for example - // when library_names[] = {libone.so, libtwo.so} and libone.so - // is loaded correctly but libtwo.so failed for some reason. - // In this case libone.so should be unloaded on return. - // See also implementation of failure_guard below. - - if (soinfos == nullptr) { - size_t soinfos_size = sizeof(soinfo*)*library_names_count; - soinfos = reinterpret_cast(alloca(soinfos_size)); - memset(soinfos, 0, soinfos_size); + load_tasks.push_back(LoadTask::create(name, nullptr)); } - // list of libraries to link - see step 2. - size_t soinfos_count = 0; + // Libraries added to this list in reverse order so that we can + // start linking from bottom-up - see step 2. + SoinfoLinkedList found_libs; + size_t soinfos_size = 0; auto failure_guard = make_scope_guard([&]() { // Housekeeping @@ -964,7 +919,7 @@ static bool find_libraries(soinfo* start_with, const char* const library_names[] LoadTask::deleter(t); }); - for (size_t i = 0; iadd_child(si); } + found_libs.push_front(si); - // When ld_preloads is not null, the first - // ld_preloads_count libs are in fact ld_preloads. - if (ld_preloads != nullptr && soinfos_count < ld_preloads_count) { - ld_preloads[soinfos_count] = si; + // When ld_preloads is not null first + // ld_preloads_size libs are in fact ld_preloads. + if (ld_preloads != nullptr && soinfos_size < ld_preloads_size) { + ld_preloads[soinfos_size] = si; } - if (soinfos_count < library_names_count) { - soinfos[soinfos_count++] = si; + if (soinfos_sizeflags & FLAG_LINKED) == 0) { - if (!si->LinkImage(local_group, extinfo)) { + if (!si->LinkImage(extinfo)) { return false; } si->flags |= FLAG_LINKED; } - - return true; - }); - - if (linked) { - failure_guard.disable(); } - return linked; + // All is well - found_libs and load_tasks are empty at this point + // and all libs are successfully linked. + failure_guard.disable(); + return true; } static soinfo* find_library(const char* name, int rtld_flags, const android_dlextinfo* extinfo) { @@ -1034,7 +979,7 @@ static soinfo* find_library(const char* name, int rtld_flags, const android_dlex soinfo* si; - if (!find_libraries(nullptr, &name, 1, &si, nullptr, 0, rtld_flags, extinfo)) { + if (!find_libraries(&name, 1, &si, nullptr, 0, rtld_flags, extinfo)) { return nullptr; } @@ -1145,7 +1090,7 @@ static ElfW(Addr) call_ifunc_resolver(ElfW(Addr) resolver_addr) { } #if defined(USE_RELA) -int soinfo::Relocate(ElfW(Rela)* rela, unsigned count, const soinfo_list_t& local_group) { +int soinfo::Relocate(ElfW(Rela)* rela, unsigned count) { for (size_t idx = 0; idx < count; ++idx, ++rela) { unsigned type = ELFW(R_TYPE)(rela->r_info); unsigned sym = ELFW(R_SYM)(rela->r_info); @@ -1163,7 +1108,7 @@ int soinfo::Relocate(ElfW(Rela)* rela, unsigned count, const soinfo_list_t& loca if (sym != 0) { sym_name = get_string(symtab[sym].st_name); - s = soinfo_do_lookup(this, sym_name, &lsi, local_group); + s = soinfo_do_lookup(this, sym_name, &lsi); if (s == nullptr) { // We only allow an undefined symbol if this is a weak reference... s = &symtab[sym]; @@ -1422,7 +1367,7 @@ int soinfo::Relocate(ElfW(Rela)* rela, unsigned count, const soinfo_list_t& loca } #else // REL, not RELA. -int soinfo::Relocate(ElfW(Rel)* rel, unsigned count, const soinfo_list_t& local_group) { +int soinfo::Relocate(ElfW(Rel)* rel, unsigned count) { for (size_t idx = 0; idx < count; ++idx, ++rel) { unsigned type = ELFW(R_TYPE)(rel->r_info); // TODO: don't use unsigned for 'sym'. Use uint32_t or ElfW(Addr) instead. @@ -1441,7 +1386,7 @@ int soinfo::Relocate(ElfW(Rel)* rel, unsigned count, const soinfo_list_t& local_ if (sym != 0) { sym_name = get_string(symtab[sym].st_name); - s = soinfo_do_lookup(this, sym_name, &lsi, local_group); + s = soinfo_do_lookup(this, sym_name, &lsi); if (s == nullptr) { // We only allow an undefined symbol if this is a weak reference... s = &symtab[sym]; @@ -1627,7 +1572,7 @@ int soinfo::Relocate(ElfW(Rel)* rel, unsigned count, const soinfo_list_t& local_ #endif #if defined(__mips__) -static bool mips_relocate_got(soinfo* si, const soinfo::soinfo_list_t& local_group) { +static bool mips_relocate_got(soinfo* si) { ElfW(Addr)** got = si->plt_got; if (got == nullptr) { return true; @@ -1660,7 +1605,7 @@ static bool mips_relocate_got(soinfo* si, const soinfo::soinfo_list_t& local_gro // This is an undefined reference... try to locate it. const char* sym_name = si->get_string(sym->st_name); soinfo* lsi = nullptr; - ElfW(Sym)* s = soinfo_do_lookup(si, sym_name, &lsi, local_group); + ElfW(Sym)* s = soinfo_do_lookup(si, sym_name, &lsi); if (s == nullptr) { // We only allow an undefined symbol if this is a weak reference. s = &symtab[g]; @@ -2253,7 +2198,7 @@ bool soinfo::PrelinkImage() { return true; } -bool soinfo::LinkImage(const soinfo_list_t& local_group, const android_dlextinfo* extinfo) { +bool soinfo::LinkImage(const android_dlextinfo* extinfo) { #if !defined(__LP64__) if (has_text_relocations) { @@ -2272,26 +2217,26 @@ bool soinfo::LinkImage(const soinfo_list_t& local_group, const android_dlextinfo #if defined(USE_RELA) if (rela != nullptr) { DEBUG("[ relocating %s ]", name); - if (Relocate(rela, rela_count, local_group)) { + if (Relocate(rela, rela_count)) { return false; } } if (plt_rela != nullptr) { DEBUG("[ relocating %s plt ]", name); - if (Relocate(plt_rela, plt_rela_count, local_group)) { + if (Relocate(plt_rela, plt_rela_count)) { return false; } } #else if (rel != nullptr) { DEBUG("[ relocating %s ]", name); - if (Relocate(rel, rel_count, local_group)) { + if (Relocate(rel, rel_count)) { return false; } } if (plt_rel != nullptr) { DEBUG("[ relocating %s plt ]", name); - if (Relocate(plt_rel, plt_rel_count, local_group)) { + if (Relocate(plt_rel, plt_rel_count)) { return false; } } @@ -2365,7 +2310,7 @@ static void add_vdso(KernelArgumentBlock& args __unused) { si->load_bias = get_elf_exec_load_bias(ehdr_vdso); si->PrelinkImage(); - si->LinkImage(g_empty_list, nullptr); + si->LinkImage(nullptr); #endif } @@ -2511,11 +2456,21 @@ static ElfW(Addr) __linker_init_post_relocation(KernelArgumentBlock& args, ElfW( }); const char* needed_library_names[needed_libraries_count]; + soinfo* needed_library_si[needed_libraries_count]; memset(needed_library_names, 0, sizeof(needed_library_names)); needed_library_name_list.copy_to_array(needed_library_names, needed_libraries_count); - if (needed_libraries_count > 0 && !find_libraries(si, needed_library_names, needed_libraries_count, nullptr, g_ld_preloads, ld_preloads_count, RTLD_GLOBAL, nullptr)) { + if (needed_libraries_count > 0 && !find_libraries(needed_library_names, needed_libraries_count, needed_library_si, g_ld_preloads, ld_preloads_count, RTLD_GLOBAL, nullptr)) { + __libc_format_fd(2, "CANNOT LINK EXECUTABLE DEPENDENCIES: %s\n", linker_get_error_buffer()); + exit(EXIT_FAILURE); + } + + for (size_t i = 0; iadd_child(needed_library_si[i]); + } + + if (!si->LinkImage(nullptr)) { __libc_format_fd(2, "CANNOT LINK EXECUTABLE: %s\n", linker_get_error_buffer()); exit(EXIT_FAILURE); } @@ -2639,7 +2594,7 @@ extern "C" ElfW(Addr) __linker_init(void* raw_args) { linker_so.phnum = elf_hdr->e_phnum; linker_so.flags |= FLAG_LINKER; - if (!(linker_so.PrelinkImage() && linker_so.LinkImage(g_empty_list, nullptr))) { + if (!(linker_so.PrelinkImage() && linker_so.LinkImage(nullptr))) { // It would be nice to print an error message, but if the linker // can't link itself, there's no guarantee that we'll be able to // call write() (because it involves a GOT reference). We may as diff --git a/linker/linker.h b/linker/linker.h index 222aca11e..ebb4793af 100644 --- a/linker/linker.h +++ b/linker/linker.h @@ -207,7 +207,7 @@ struct soinfo { void CallDestructors(); void CallPreInitConstructors(); bool PrelinkImage(); - bool LinkImage(const soinfo_list_t& local_group, const android_dlextinfo* extinfo); + bool LinkImage(const android_dlextinfo* extinfo); void add_child(soinfo* child); void remove_all_links(); @@ -234,9 +234,9 @@ struct soinfo { void CallArray(const char* array_name, linker_function_t* functions, size_t count, bool reverse); void CallFunction(const char* function_name, linker_function_t function); #if defined(USE_RELA) - int Relocate(ElfW(Rela)* rela, unsigned count, const soinfo_list_t& local_group); + int Relocate(ElfW(Rela)* rela, unsigned count); #else - int Relocate(ElfW(Rel)* rel, unsigned count, const soinfo_list_t& local_group); + int Relocate(ElfW(Rel)* rel, unsigned count); #endif private: diff --git a/tests/dlfcn_test.cpp b/tests/dlfcn_test.cpp index e604f5abe..f1ec0f131 100644 --- a/tests/dlfcn_test.cpp +++ b/tests/dlfcn_test.cpp @@ -162,39 +162,39 @@ TEST(dlfcn, dlopen_check_relocation_dt_needed_order) { ASSERT_EQ(1, fn()); } -TEST(dlfcn, dlopen_check_order_dlsym) { +TEST(dlfcn, dlopen_check_order) { // Here is how the test library and its dt_needed // libraries are arranged // - // libtest_check_order_children.so + // libtest_check_order.so // | - // +-> ..._1_left.so + // +-> libtest_check_order_1_left.so // | | - // | +-> ..._a.so + // | +-> libtest_check_order_a.so // | | - // | +-> ...r_b.so + // | +-> libtest_check_order_b.so // | - // +-> ..._2_right.so + // +-> libtest_check_order_2_right.so // | | - // | +-> ..._d.so + // | +-> libtest_check_order_d.so // | | - // | +-> ..._b.so + // | +-> libtest_check_order_b.so // | - // +-> ..._3_c.so + // +-> libtest_check_order_3_c.so // // load order should be (1, 2, 3, a, b, d) // // get_answer() is defined in (2, 3, a, b, c) // get_answer2() is defined in (b, d) - void* sym = dlsym(RTLD_DEFAULT, "check_order_dlsym_get_answer"); + void* sym = dlsym(RTLD_DEFAULT, "dlopen_test_get_answer"); ASSERT_TRUE(sym == nullptr); - void* handle = dlopen("libtest_check_order_dlsym.so", RTLD_NOW | RTLD_GLOBAL); - ASSERT_TRUE(handle != nullptr) << dlerror(); + void* handle = dlopen("libtest_check_order.so", RTLD_NOW | RTLD_GLOBAL); + ASSERT_TRUE(handle != nullptr); typedef int (*fn_t) (void); fn_t fn, fn2; - fn = reinterpret_cast(dlsym(RTLD_DEFAULT, "check_order_dlsym_get_answer")); + fn = reinterpret_cast(dlsym(RTLD_DEFAULT, "dlopen_test_get_answer")); ASSERT_TRUE(fn != NULL) << dlerror(); - fn2 = reinterpret_cast(dlsym(RTLD_DEFAULT, "check_order_dlsym_get_answer2")); + fn2 = reinterpret_cast(dlsym(RTLD_DEFAULT, "dlopen_test_get_answer2")); ASSERT_TRUE(fn2 != NULL) << dlerror(); ASSERT_EQ(42, fn()); @@ -202,163 +202,6 @@ TEST(dlfcn, dlopen_check_order_dlsym) { dlclose(handle); } -TEST(dlfcn, dlopen_check_order_reloc_siblings) { - // This is how this one works: - // we lookup and call get_answer which is defined in '_2.so' - // and in turn calls external get_answer_impl() defined in _1.so and in '_[a-f].so' - // the correct _impl() is implemented by '_a.so'; - // - // Note that this is test for RTLD_LOCAL (TODO: test for GLOBAL?) - // - // Here is the picture: - // - // libtest_check_order_reloc_siblings.so - // | - // +-> ..._1.so <- empty - // | | - // | +-> ..._a.so <- exports correct answer_impl() - // | | - // | +-> ..._b.so <- every other letter exporting incorrect one. - // | - // +-> ..._2.so <- empty - // | | - // | +-> ..._c.so - // | | - // | +-> ..._d.so - // | - // +-> ..._3.so <- empty - // | - // +-> ..._e.so - // | - // +-> ..._f.so <- exports get_answer() that calls get_anser_impl(); - // implements incorrect get_answer_impl() - - void* handle = dlopen("libtest_check_order_reloc_siblings.so", RTLD_NOW | RTLD_NOLOAD); - ASSERT_TRUE(handle == nullptr); -#ifdef __BIONIC__ - // TODO: glibc returns nullptr on dlerror() here. Is it bug? - ASSERT_STREQ("dlopen failed: library \"libtest_check_order_reloc_siblings.so\" wasn't loaded and RTLD_NOLOAD prevented it", dlerror()); -#endif - - handle = dlopen("libtest_check_order_reloc_siblings.so", RTLD_NOW | RTLD_LOCAL); - ASSERT_TRUE(handle != nullptr) << dlerror(); - - typedef int (*fn_t) (void); - fn_t fn = reinterpret_cast(dlsym(handle, "check_order_reloc_get_answer")); - ASSERT_TRUE(fn != nullptr) << dlerror(); - ASSERT_EQ(42, fn()); - - ASSERT_EQ(0, dlclose(handle)); -} - -TEST(dlfcn, dlopen_check_order_reloc_siblings_with_preload) { - // This test uses the same library as dlopen_check_order_reloc_siblings. - // Unlike dlopen_check_order_reloc_siblings it preloads - // libtest_check_order_reloc_siblings_1.so (first dependency) prior to - // dlopen(libtest_check_order_reloc_siblings.so) - - void* handle = dlopen("libtest_check_order_reloc_siblings.so", RTLD_NOW | RTLD_NOLOAD); - ASSERT_TRUE(handle == nullptr); - handle = dlopen("libtest_check_order_reloc_siblings_1.so", RTLD_NOW | RTLD_NOLOAD); - ASSERT_TRUE(handle == nullptr); - - void* handle_for_1 = dlopen("libtest_check_order_reloc_siblings_1.so", RTLD_NOW | RTLD_LOCAL); - ASSERT_TRUE(handle_for_1 != nullptr) << dlerror(); - - handle = dlopen("libtest_check_order_reloc_siblings.so", RTLD_NOW | RTLD_LOCAL); - ASSERT_TRUE(handle != nullptr) << dlerror(); - - ASSERT_EQ(0, dlclose(handle_for_1)); - - typedef int (*fn_t) (void); - fn_t fn = reinterpret_cast(dlsym(handle, "check_order_reloc_get_answer")); - ASSERT_TRUE(fn != nullptr) << dlerror(); - ASSERT_EQ(42, fn()); - - ASSERT_EQ(0, dlclose(handle)); -} - -TEST(dlfcn, dlopen_check_order_reloc_nephew) { - // This is how this one works: - // we lookup and call nephew_get_answer which is defined in '_2.so' - // and in turn calls external get_answer_impl() defined in '_[a-f].so' - // the correct _impl() is implemented by '_a.so'; - // - // Here is the picture: - // - // libtest_check_order_reloc_siblings.so - // | - // +-> ..._1.so <- empty - // | | - // | +-> ..._a.so <- exports correct answer_impl() - // | | - // | +-> ..._b.so <- every other letter exporting incorrect one. - // | - // +-> ..._2.so <- empty - // | | - // | +-> ..._c.so - // | | - // | +-> ..._d.so - // | - // +-> ..._3.so <- nephew_get_answer() that calls get_answer_impl(); - // | - // +-> ..._e.so - // | - // +-> ..._f.so - - void* handle = dlopen("libtest_check_order_reloc_siblings.so", RTLD_NOW | RTLD_NOLOAD); - ASSERT_TRUE(handle == nullptr); -#ifdef __BIONIC__ - // TODO: glibc returns nullptr on dlerror() here. Is it bug? - ASSERT_STREQ("dlopen failed: library \"libtest_check_order_reloc_siblings.so\" wasn't loaded and RTLD_NOLOAD prevented it", dlerror()); -#endif - - handle = dlopen("libtest_check_order_reloc_siblings.so", RTLD_NOW | RTLD_LOCAL); - ASSERT_TRUE(handle != nullptr) << dlerror(); - - typedef int (*fn_t) (void); - fn_t fn = reinterpret_cast(dlsym(handle, "check_order_reloc_nephew_get_answer")); - ASSERT_TRUE(fn != nullptr) << dlerror(); - ASSERT_EQ(42, fn()); - - ASSERT_EQ(0, dlclose(handle)); -} - -extern "C" int check_order_reloc_root_get_answer_impl() { - return 42; -} - -TEST(dlfcn, dlopen_check_order_reloc_main_executable) { - // This is how this one works: - // we lookup and call get_answer3 which is defined in 'root.so' - // and in turn calls external root_get_answer_impl() defined in _2.so and - // above the correct _impl() is one in the executable. - // - // libtest_check_order_reloc_root.so - // | - // +-> ..._1.so <- empty - // | - // +-> ..._2.so <- gives incorrect answer for answer_main_impl() - // - - void* handle = dlopen("libtest_check_order_reloc_root.so", RTLD_NOW | RTLD_NOLOAD); - ASSERT_TRUE(handle == nullptr); -#ifdef __BIONIC__ - // TODO: glibc returns nullptr on dlerror() here. Is it bug? - ASSERT_STREQ("dlopen failed: library \"libtest_check_order_reloc_root.so\" wasn't loaded and RTLD_NOLOAD prevented it", dlerror()); -#endif - - handle = dlopen("libtest_check_order_reloc_root.so", RTLD_NOW | RTLD_LOCAL); - ASSERT_TRUE(handle != nullptr) << dlerror(); - - typedef int (*fn_t) (void); - fn_t fn = reinterpret_cast(dlsym(handle, "check_order_reloc_root_get_answer")); - ASSERT_TRUE(fn != nullptr) << dlerror(); - ASSERT_EQ(42, fn()); - - ASSERT_EQ(0, dlclose(handle)); -} - TEST(dlfcn, dlopen_check_rtld_local) { void* sym = dlsym(RTLD_DEFAULT, "dlopen_testlib_simple_func"); ASSERT_TRUE(sym == nullptr); @@ -499,6 +342,7 @@ TEST(dlfcn, dlopen_nodelete_dt_flags_1) { ASSERT_TRUE(!is_unloaded); } + TEST(dlfcn, dlopen_failure) { void* self = dlopen("/does/not/exist", RTLD_NOW); ASSERT_TRUE(self == NULL); diff --git a/tests/libs/Android.build.dlopen_check_order_dlsym.mk b/tests/libs/Android.build.dlopen_check_order_dlsym.mk deleted file mode 100644 index 73d8c1a83..000000000 --- a/tests/libs/Android.build.dlopen_check_order_dlsym.mk +++ /dev/null @@ -1,90 +0,0 @@ -# -# Copyright (C) 2012 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# ----------------------------------------------------------------------------- -# Libraries used by dlfcn tests to verify correct load order: -# libtest_check_order_2_right.so -# ----------------------------------------------------------------------------- -libtest_check_order_dlsym_2_right_src_files := \ - dlopen_check_order_dlsym_answer.cpp - -libtest_check_order_dlsym_2_right_cflags := -D__ANSWER=42 -module := libtest_check_order_dlsym_2_right -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# libtest_check_order_a.so -# ----------------------------------------------------------------------------- -libtest_check_order_dlsym_a_src_files := \ - dlopen_check_order_dlsym_answer.cpp - -libtest_check_order_dlsym_a_cflags := -D__ANSWER=1 -module := libtest_check_order_dlsym_a -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# libtest_check_order_b.so -# ----------------------------------------------------------------------------- -libtest_check_order_dlsym_b_src_files := \ - dlopen_check_order_dlsym_answer.cpp - -libtest_check_order_dlsym_b_cflags := -D__ANSWER=2 -D__ANSWER2=43 -module := libtest_check_order_dlsym_b -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# libtest_check_order_c.so -# ----------------------------------------------------------------------------- -libtest_check_order_dlsym_3_c_src_files := \ - dlopen_check_order_dlsym_answer.cpp - -libtest_check_order_dlsym_3_c_cflags := -D__ANSWER=3 -module := libtest_check_order_dlsym_3_c -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# libtest_check_order_d.so -# ----------------------------------------------------------------------------- -libtest_check_order_dlsym_d_src_files := \ - dlopen_check_order_dlsym_answer.cpp - -libtest_check_order_dlsym_d_shared_libraries := libtest_check_order_dlsym_b -libtest_check_order_dlsym_d_cflags := -D__ANSWER=4 -D__ANSWER2=4 -module := libtest_check_order_dlsym_d -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# libtest_check_order_left.so -# ----------------------------------------------------------------------------- -libtest_check_order_dlsym_1_left_src_files := \ - empty.cpp - -libtest_check_order_dlsym_1_left_shared_libraries := libtest_check_order_dlsym_a libtest_check_order_dlsym_b - -module := libtest_check_order_dlsym_1_left -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# libtest_check_order.so -# ----------------------------------------------------------------------------- -libtest_check_order_dlsym_src_files := \ - empty.cpp - -libtest_check_order_dlsym_shared_libraries := libtest_check_order_dlsym_1_left \ - libtest_check_order_dlsym_2_right libtest_check_order_dlsym_3_c - -module := libtest_check_order_dlsym -include $(LOCAL_PATH)/Android.build.testlib.mk diff --git a/tests/libs/Android.build.dlopen_check_order_reloc_main_executable.mk b/tests/libs/Android.build.dlopen_check_order_reloc_main_executable.mk deleted file mode 100644 index 639696b25..000000000 --- a/tests/libs/Android.build.dlopen_check_order_reloc_main_executable.mk +++ /dev/null @@ -1,56 +0,0 @@ -# -# Copyright (C) 2012 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# ----------------------------------------------------------------------------- -# Libraries used by dlfcn tests to verify correct relocation order: -# libtest_check_order_reloc_root*.so -# ----------------------------------------------------------------------------- - - -# ----------------------------------------------------------------------------- -# ..._1.so - empty -# ----------------------------------------------------------------------------- -libtest_check_order_reloc_root_1_src_files := \ - empty.cpp - - -module := libtest_check_order_reloc_root_1 -include $(LOCAL_PATH)/Android.build.testlib.mk - - -# ----------------------------------------------------------------------------- -# ..._2.so - this one has the incorrect answer -# ----------------------------------------------------------------------------- -libtest_check_order_reloc_root_2_src_files := \ - dlopen_check_order_reloc_root_answer_impl.cpp - -libtest_check_order_reloc_root_2_cflags := -D__ANSWER=2 - -module := libtest_check_order_reloc_root_2 -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# libtest_check_order_reloc_root.so <- implements get_answer3() -# ----------------------------------------------------------------------------- -libtest_check_order_reloc_root_src_files := \ - dlopen_check_order_reloc_root_answer.cpp - -libtest_check_order_reloc_root_shared_libraries := \ - libtest_check_order_reloc_root_1 \ - libtest_check_order_reloc_root_2 - -module := libtest_check_order_reloc_root -include $(LOCAL_PATH)/Android.build.testlib.mk diff --git a/tests/libs/Android.build.dlopen_check_order_reloc_siblings.mk b/tests/libs/Android.build.dlopen_check_order_reloc_siblings.mk deleted file mode 100644 index 0f1a2b4da..000000000 --- a/tests/libs/Android.build.dlopen_check_order_reloc_siblings.mk +++ /dev/null @@ -1,133 +0,0 @@ -# -# Copyright (C) 2014 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# ----------------------------------------------------------------------------- -# Libraries used by dlfcn tests to verify correct relocation order: -# libtest_check_order_reloc_siblings*.so -# ----------------------------------------------------------------------------- - -# ----------------------------------------------------------------------------- -# ..._1.so - empty -# ----------------------------------------------------------------------------- -libtest_check_order_reloc_siblings_1_src_files := \ - empty.cpp - -libtest_check_order_reloc_siblings_1_shared_libraries := \ - libtest_check_order_reloc_siblings_a \ - libtest_check_order_reloc_siblings_b - -module := libtest_check_order_reloc_siblings_1 -include $(LOCAL_PATH)/Android.build.testlib.mk - - -# ----------------------------------------------------------------------------- -# ..._2.so - empty -# ----------------------------------------------------------------------------- -libtest_check_order_reloc_siblings_2_src_files := \ - empty.cpp - -libtest_check_order_reloc_siblings_2_shared_libraries := \ - libtest_check_order_reloc_siblings_c \ - libtest_check_order_reloc_siblings_d - -module := libtest_check_order_reloc_siblings_2 -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# ..._3.so - get_answer2(); -# ----------------------------------------------------------------------------- -libtest_check_order_reloc_siblings_3_src_files := \ - dlopen_check_order_reloc_nephew_answer.cpp - -libtest_check_order_reloc_siblings_3_shared_libraries := \ - libtest_check_order_reloc_siblings_e \ - libtest_check_order_reloc_siblings_f - -module := libtest_check_order_reloc_siblings_3 -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# ..._a.so <- correct impl -# ----------------------------------------------------------------------------- -libtest_check_order_reloc_siblings_a_src_files := \ - dlopen_check_order_reloc_answer_impl.cpp - -libtest_check_order_reloc_siblings_a_cflags := -D__ANSWER=42 -module := libtest_check_order_reloc_siblings_a -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# ..._b.so -# ----------------------------------------------------------------------------- -libtest_check_order_reloc_siblings_b_src_files := \ - dlopen_check_order_reloc_answer_impl.cpp - -libtest_check_order_reloc_siblings_b_cflags := -D__ANSWER=1 -module := libtest_check_order_reloc_siblings_b -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# ..._c.so -# ----------------------------------------------------------------------------- -libtest_check_order_reloc_siblings_c_src_files := \ - dlopen_check_order_reloc_answer_impl.cpp - -libtest_check_order_reloc_siblings_c_cflags := -D__ANSWER=2 -module := libtest_check_order_reloc_siblings_c -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# ..._d.so -# ----------------------------------------------------------------------------- -libtest_check_order_reloc_siblings_d_src_files := \ - dlopen_check_order_reloc_answer_impl.cpp - -libtest_check_order_reloc_siblings_d_cflags := -D__ANSWER=3 -module := libtest_check_order_reloc_siblings_d -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# ..._e.so -# ----------------------------------------------------------------------------- -libtest_check_order_reloc_siblings_e_src_files := \ - dlopen_check_order_reloc_answer_impl.cpp - -libtest_check_order_reloc_siblings_e_cflags := -D__ANSWER=4 -module := libtest_check_order_reloc_siblings_e -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# ..._f.so <- get_answer() -# ----------------------------------------------------------------------------- -libtest_check_order_reloc_siblings_f_src_files := \ - dlopen_check_order_reloc_answer.cpp - -module := libtest_check_order_reloc_siblings_f -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# libtest_check_order_reloc_siblings.so -# ----------------------------------------------------------------------------- -libtest_check_order_reloc_siblings_src_files := \ - empty.cpp - -libtest_check_order_reloc_siblings_shared_libraries := \ - libtest_check_order_reloc_siblings_1 \ - libtest_check_order_reloc_siblings_2 \ - libtest_check_order_reloc_siblings_3 - -module := libtest_check_order_reloc_siblings -include $(LOCAL_PATH)/Android.build.testlib.mk diff --git a/tests/libs/Android.mk b/tests/libs/Android.mk index 05e7113ba..175a63539 100644 --- a/tests/libs/Android.mk +++ b/tests/libs/Android.mk @@ -21,9 +21,6 @@ common_cppflags += -std=gnu++11 common_additional_dependencies := \ $(LOCAL_PATH)/Android.mk \ $(LOCAL_PATH)/Android.build.dlext_testzip.mk \ - $(LOCAL_PATH)/Android.build.dlopen_check_order_dlsym.mk \ - $(LOCAL_PATH)/Android.build.dlopen_check_order_reloc_siblings.mk \ - $(LOCAL_PATH)/Android.build.dlopen_check_order_reloc_main_executable.mk \ $(LOCAL_PATH)/Android.build.testlib.mk \ $(TEST_PATH)/Android.build.mk @@ -153,19 +150,79 @@ module := libtest_nodelete_dt_flags_1 include $(LOCAL_PATH)/Android.build.testlib.mk # ----------------------------------------------------------------------------- -# Build libtest_check_order_dlsym.so with its dependencies. +# Libraries used by dlfcn tests to verify correct load order: +# libtest_check_order_2_right.so # ----------------------------------------------------------------------------- -include $(LOCAL_PATH)/Android.build.dlopen_check_order_dlsym.mk +libtest_check_order_2_right_src_files := \ + dlopen_testlib_answer.cpp + +libtest_check_order_2_right_cflags := -D__ANSWER=42 +module := libtest_check_order_2_right +include $(LOCAL_PATH)/Android.build.testlib.mk + +# ----------------------------------------------------------------------------- +# libtest_check_order_a.so +# ----------------------------------------------------------------------------- +libtest_check_order_a_src_files := \ + dlopen_testlib_answer.cpp + +libtest_check_order_a_cflags := -D__ANSWER=1 +module := libtest_check_order_a +include $(LOCAL_PATH)/Android.build.testlib.mk + +# ----------------------------------------------------------------------------- +# libtest_check_order_b.so +# ----------------------------------------------------------------------------- +libtest_check_order_b_src_files := \ + dlopen_testlib_answer.cpp + +libtest_check_order_b_cflags := -D__ANSWER=2 -D__ANSWER2=43 +module := libtest_check_order_b +include $(LOCAL_PATH)/Android.build.testlib.mk + +# ----------------------------------------------------------------------------- +# libtest_check_order_c.so +# ----------------------------------------------------------------------------- +libtest_check_order_3_c_src_files := \ + dlopen_testlib_answer.cpp + +libtest_check_order_3_c_cflags := -D__ANSWER=3 +module := libtest_check_order_3_c +include $(LOCAL_PATH)/Android.build.testlib.mk + +# ----------------------------------------------------------------------------- +# libtest_check_order_d.so +# ----------------------------------------------------------------------------- +libtest_check_order_d_src_files := \ + dlopen_testlib_answer.cpp + +libtest_check_order_d_shared_libraries := libtest_check_order_b +libtest_check_order_d_cflags := -D__ANSWER=4 -D__ANSWER2=4 +module := libtest_check_order_d +include $(LOCAL_PATH)/Android.build.testlib.mk # ----------------------------------------------------------------------------- -# Build libtest_check_order_siblings.so with its dependencies. +# libtest_check_order_left.so # ----------------------------------------------------------------------------- -include $(LOCAL_PATH)/Android.build.dlopen_check_order_reloc_siblings.mk +libtest_check_order_1_left_src_files := \ + empty.cpp + +libtest_check_order_1_left_shared_libraries := libtest_check_order_a libtest_check_order_b + +module := libtest_check_order_1_left +include $(LOCAL_PATH)/Android.build.testlib.mk # ----------------------------------------------------------------------------- -# Build libtest_check_order_root.so with its dependencies. +# libtest_check_order.so # ----------------------------------------------------------------------------- -include $(LOCAL_PATH)/Android.build.dlopen_check_order_reloc_main_executable.mk +libtest_check_order_src_files := \ + empty.cpp + +libtest_check_order_shared_libraries := libtest_check_order_1_left \ + libtest_check_order_2_right libtest_check_order_3_c + +module := libtest_check_order +include $(LOCAL_PATH)/Android.build.testlib.mk # ----------------------------------------------------------------------------- # Library with dependency loop used by dlfcn tests diff --git a/tests/libs/dlopen_check_order_reloc_answer.cpp b/tests/libs/dlopen_check_order_reloc_answer.cpp deleted file mode 100644 index 036670bee..000000000 --- a/tests/libs/dlopen_check_order_reloc_answer.cpp +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -extern "C" int __attribute__((weak)) check_order_reloc_get_answer_impl() { - return 0; -} - -extern "C" int check_order_reloc_get_answer() { - return check_order_reloc_get_answer_impl(); -} diff --git a/tests/libs/dlopen_check_order_reloc_answer_impl.cpp b/tests/libs/dlopen_check_order_reloc_answer_impl.cpp deleted file mode 100644 index 324b905da..000000000 --- a/tests/libs/dlopen_check_order_reloc_answer_impl.cpp +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -extern "C" int check_order_reloc_get_answer_impl() { - return __ANSWER; -} diff --git a/tests/libs/dlopen_check_order_reloc_nephew_answer.cpp b/tests/libs/dlopen_check_order_reloc_nephew_answer.cpp deleted file mode 100644 index 065d1bef7..000000000 --- a/tests/libs/dlopen_check_order_reloc_nephew_answer.cpp +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -extern "C" int check_order_reloc_get_answer_impl(); - -extern "C" int check_order_reloc_nephew_get_answer() { - return check_order_reloc_get_answer_impl(); -} diff --git a/tests/libs/dlopen_check_order_reloc_root_answer.cpp b/tests/libs/dlopen_check_order_reloc_root_answer.cpp deleted file mode 100644 index b21abd77a..000000000 --- a/tests/libs/dlopen_check_order_reloc_root_answer.cpp +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -extern "C" int check_order_reloc_root_get_answer_impl(); - -extern "C" int check_order_reloc_root_get_answer() { - return check_order_reloc_root_get_answer_impl(); -} diff --git a/tests/libs/dlopen_check_order_reloc_root_answer_impl.cpp b/tests/libs/dlopen_check_order_reloc_root_answer_impl.cpp deleted file mode 100644 index 25fb9aca9..000000000 --- a/tests/libs/dlopen_check_order_reloc_root_answer_impl.cpp +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -extern "C" int check_order_reloc_root_get_answer_impl() { - return __ANSWER; -} diff --git a/tests/libs/dlopen_check_order_dlsym_answer.cpp b/tests/libs/dlopen_testlib_answer.cpp similarity index 87% rename from tests/libs/dlopen_check_order_dlsym_answer.cpp rename to tests/libs/dlopen_testlib_answer.cpp index 2ae6cf79e..a4d75046e 100644 --- a/tests/libs/dlopen_check_order_dlsym_answer.cpp +++ b/tests/libs/dlopen_testlib_answer.cpp @@ -14,12 +14,12 @@ * limitations under the License. */ -extern "C" int check_order_dlsym_get_answer() { +extern "C" int dlopen_test_get_answer() { return __ANSWER; } #ifdef __ANSWER2 -extern "C" int check_order_dlsym_get_answer2() { +extern "C" int dlopen_test_get_answer2() { return __ANSWER2; } #endif From d84897d4a30d5521e7f16d986142b634ef28541a Mon Sep 17 00:00:00 2001 From: Dmitriy Ivanov Date: Mon, 3 Nov 2014 22:08:31 -0800 Subject: [PATCH 21/89] Revert "Add RTLD_NODELETE flag support" This reverts commit c87f65d2cd0690d81665f8b241c1d763f72b6f80. Bug: 18222321 Bug: 18211780 Change-Id: I00252e26a28a41ab9f1e2dd3b32f0f80d86297f1 --- libc/include/dlfcn.h | 1 - linker/linker.cpp | 16 +---- linker/linker.h | 18 +++-- tests/dlfcn_test.cpp | 80 ----------------------- tests/libs/Android.mk | 29 -------- tests/libs/dlopen_nodelete_1.cpp | 31 --------- tests/libs/dlopen_nodelete_2.cpp | 31 --------- tests/libs/dlopen_nodelete_dt_flags_1.cpp | 30 --------- 8 files changed, 10 insertions(+), 226 deletions(-) delete mode 100644 tests/libs/dlopen_nodelete_1.cpp delete mode 100644 tests/libs/dlopen_nodelete_2.cpp delete mode 100644 tests/libs/dlopen_nodelete_dt_flags_1.cpp diff --git a/libc/include/dlfcn.h b/libc/include/dlfcn.h index afa76878f..8dde08cf5 100644 --- a/libc/include/dlfcn.h +++ b/libc/include/dlfcn.h @@ -64,7 +64,6 @@ enum { RTLD_GLOBAL = 2, #endif RTLD_NOLOAD = 4, - RTLD_NODELETE = 0x01000, }; #if defined (__LP64__) diff --git a/linker/linker.cpp b/linker/linker.cpp index 41557e231..636541297 100644 --- a/linker/linker.cpp +++ b/linker/linker.cpp @@ -987,11 +987,6 @@ static soinfo* find_library(const char* name, int rtld_flags, const android_dlex } static void soinfo_unload(soinfo* si) { - if (!si->can_unload()) { - TRACE("not unloading '%s' - the binary is flagged with NODELETE", si->name); - return; - } - if (si->ref_count == 1) { TRACE("unloading '%s'", si->name); si->CallDestructors(); @@ -1050,7 +1045,7 @@ void do_android_update_LD_LIBRARY_PATH(const char* ld_library_path) { } soinfo* do_dlopen(const char* name, int flags, const android_dlextinfo* extinfo) { - if ((flags & ~(RTLD_NOW|RTLD_LAZY|RTLD_LOCAL|RTLD_GLOBAL|RTLD_NODELETE|RTLD_NOLOAD)) != 0) { + if ((flags & ~(RTLD_NOW|RTLD_LAZY|RTLD_LOCAL|RTLD_GLOBAL|RTLD_NOLOAD)) != 0) { DL_ERR("invalid flags to dlopen: %x", flags); return nullptr; } @@ -1813,9 +1808,6 @@ const char* soinfo::get_string(ElfW(Word) index) const { return strtab + index; } -bool soinfo::can_unload() const { - return (rtld_flags & (RTLD_NODELETE | RTLD_GLOBAL)) == 0; -} /* Force any of the closed stdin, stdout and stderr to be associated with /dev/null. */ static int nullify_closed_stdio() { @@ -2119,13 +2111,9 @@ bool soinfo::PrelinkImage() { if ((d->d_un.d_val & DF_1_GLOBAL) != 0) { rtld_flags |= RTLD_GLOBAL; } - - if ((d->d_un.d_val & DF_1_NODELETE) != 0) { - rtld_flags |= RTLD_NODELETE; - } // TODO: Implement other flags - if ((d->d_un.d_val & ~(DF_1_NOW | DF_1_GLOBAL | DF_1_NODELETE)) != 0) { + if ((d->d_un.d_val & ~(DF_1_NOW | DF_1_GLOBAL)) != 0) { DL_WARN("Unsupported flags DT_FLAGS_1=%p", reinterpret_cast(d->d_un.d_val)); } break; diff --git a/linker/linker.h b/linker/linker.h index ebb4793af..6329efda6 100644 --- a/linker/linker.h +++ b/linker/linker.h @@ -134,7 +134,7 @@ struct soinfo { #endif soinfo* next; - uint32_t flags; + unsigned flags; private: const char* strtab; @@ -143,8 +143,8 @@ struct soinfo { size_t nbucket; size_t nchain; - uint32_t* bucket; - uint32_t* chain; + unsigned* bucket; + unsigned* chain; #if defined(__mips__) || !defined(__LP64__) // This is only used by mips and mips64, but needs to be here for @@ -179,12 +179,12 @@ struct soinfo { #if defined(__arm__) // ARM EABI section used for stack unwinding. - uint32_t* ARM_exidx; + unsigned* ARM_exidx; size_t ARM_exidx_count; #elif defined(__mips__) - uint32_t mips_symtabno; - uint32_t mips_local_gotno; - uint32_t mips_gotsym; + unsigned mips_symtabno; + unsigned mips_local_gotno; + unsigned mips_gotsym; #endif size_t ref_count; @@ -224,12 +224,10 @@ struct soinfo { ElfW(Addr) resolve_symbol_address(ElfW(Sym)* s); const char* get_string(ElfW(Word) index) const; - bool can_unload() const; bool inline has_min_version(uint32_t min_version) const { return (flags & FLAG_NEW_SOINFO) != 0 && version >= min_version; } - private: void CallArray(const char* array_name, linker_function_t* functions, size_t count, bool reverse); void CallFunction(const char* function_name, linker_function_t function); @@ -260,7 +258,7 @@ struct soinfo { friend soinfo* get_libdl_info(); }; -soinfo* get_libdl_info(); +extern soinfo* get_libdl_info(); void do_android_get_LD_LIBRARY_PATH(char*, size_t); void do_android_update_LD_LIBRARY_PATH(const char* ld_library_path); diff --git a/tests/dlfcn_test.cpp b/tests/dlfcn_test.cpp index f1ec0f131..c9c856a37 100644 --- a/tests/dlfcn_test.cpp +++ b/tests/dlfcn_test.cpp @@ -232,15 +232,10 @@ TEST(dlfcn, dlopen_check_rtld_global) { ASSERT_TRUE(sym == nullptr); void* handle = dlopen("libtest_simple.so", RTLD_NOW | RTLD_GLOBAL); - ASSERT_TRUE(handle != nullptr) << dlerror(); sym = dlsym(RTLD_DEFAULT, "dlopen_testlib_simple_func"); ASSERT_TRUE(sym != nullptr) << dlerror(); ASSERT_TRUE(reinterpret_cast(sym)()); dlclose(handle); - - // RTLD_GLOBAL implies RTLD_NODELETE, let's check that - void* sym_after_dlclose = dlsym(RTLD_DEFAULT, "dlopen_testlib_simple_func"); - ASSERT_EQ(sym, sym_after_dlclose); } // libtest_with_dependency_loop.so -> libtest_with_dependency_loop_a.so -> @@ -268,81 +263,6 @@ TEST(dlfcn, dlopen_check_loop) { #endif } -TEST(dlfcn, dlopen_nodelete) { - static bool is_unloaded = false; - - void* handle = dlopen("libtest_nodelete_1.so", RTLD_NOW | RTLD_NODELETE); - ASSERT_TRUE(handle != nullptr) << dlerror(); - void (*set_unload_flag_ptr)(bool*); - set_unload_flag_ptr = reinterpret_cast(dlsym(handle, "dlopen_nodelete_1_set_unload_flag_ptr")); - ASSERT_TRUE(set_unload_flag_ptr != nullptr) << dlerror(); - set_unload_flag_ptr(&is_unloaded); - - uint32_t* taxicab_number = reinterpret_cast(dlsym(handle, "dlopen_nodelete_1_taxicab_number")); - ASSERT_TRUE(taxicab_number != nullptr) << dlerror(); - ASSERT_EQ(1729U, *taxicab_number); - *taxicab_number = 2; - - dlclose(handle); - ASSERT_TRUE(!is_unloaded); - - uint32_t* taxicab_number_after_dlclose = reinterpret_cast(dlsym(handle, "dlopen_nodelete_1_taxicab_number")); - ASSERT_EQ(taxicab_number_after_dlclose, taxicab_number); - ASSERT_EQ(2U, *taxicab_number_after_dlclose); - - - handle = dlopen("libtest_nodelete_1.so", RTLD_NOW); - uint32_t* taxicab_number2 = reinterpret_cast(dlsym(handle, "dlopen_nodelete_1_taxicab_number")); - ASSERT_EQ(taxicab_number2, taxicab_number); - - ASSERT_EQ(2U, *taxicab_number2); - - dlclose(handle); - ASSERT_TRUE(!is_unloaded); -} - -TEST(dlfcn, dlopen_nodelete_on_second_dlopen) { - static bool is_unloaded = false; - - void* handle = dlopen("libtest_nodelete_2.so", RTLD_NOW); - ASSERT_TRUE(handle != nullptr) << dlerror(); - void (*set_unload_flag_ptr)(bool*); - set_unload_flag_ptr = reinterpret_cast(dlsym(handle, "dlopen_nodelete_2_set_unload_flag_ptr")); - ASSERT_TRUE(set_unload_flag_ptr != nullptr) << dlerror(); - set_unload_flag_ptr(&is_unloaded); - - uint32_t* taxicab_number = reinterpret_cast(dlsym(handle, "dlopen_nodelete_2_taxicab_number")); - ASSERT_TRUE(taxicab_number != nullptr) << dlerror(); - - ASSERT_EQ(1729U, *taxicab_number); - *taxicab_number = 2; - - // This RTLD_NODELETE should be ignored - void* handle1 = dlopen("libtest_nodelete_2.so", RTLD_NOW | RTLD_NODELETE); - ASSERT_TRUE(handle1 != nullptr) << dlerror(); - ASSERT_EQ(handle, handle1); - - dlclose(handle1); - dlclose(handle); - - ASSERT_TRUE(is_unloaded); -} - -TEST(dlfcn, dlopen_nodelete_dt_flags_1) { - static bool is_unloaded = false; - - void* handle = dlopen("libtest_nodelete_dt_flags_1.so", RTLD_NOW); - ASSERT_TRUE(handle != nullptr) << dlerror(); - void (*set_unload_flag_ptr)(bool*); - set_unload_flag_ptr = reinterpret_cast(dlsym(handle, "dlopen_nodelete_dt_flags_1_set_unload_flag_ptr")); - ASSERT_TRUE(set_unload_flag_ptr != nullptr) << dlerror(); - set_unload_flag_ptr(&is_unloaded); - - dlclose(handle); - ASSERT_TRUE(!is_unloaded); -} - - TEST(dlfcn, dlopen_failure) { void* self = dlopen("/does/not/exist", RTLD_NOW); ASSERT_TRUE(self == NULL); diff --git a/tests/libs/Android.mk b/tests/libs/Android.mk index 175a63539..af3e070a8 100644 --- a/tests/libs/Android.mk +++ b/tests/libs/Android.mk @@ -120,35 +120,6 @@ libtest_simple_src_files := \ module := libtest_simple include $(LOCAL_PATH)/Android.build.testlib.mk -# ----------------------------------------------------------------------------- -# Library used by dlfcn nodelete tests -# ----------------------------------------------------------------------------- -libtest_nodelete_1_src_files := \ - dlopen_nodelete_1.cpp - -module := libtest_nodelete_1 -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# Library used by dlfcn nodelete tests -# ----------------------------------------------------------------------------- -libtest_nodelete_2_src_files := \ - dlopen_nodelete_2.cpp - -module := libtest_nodelete_2 -include $(LOCAL_PATH)/Android.build.testlib.mk - -# ----------------------------------------------------------------------------- -# Library used by dlfcn nodelete tests -# ----------------------------------------------------------------------------- -libtest_nodelete_dt_flags_1_src_files := \ - dlopen_nodelete_dt_flags_1.cpp - -libtest_nodelete_dt_flags_1_ldflags := -Wl,-z,nodelete - -module := libtest_nodelete_dt_flags_1 -include $(LOCAL_PATH)/Android.build.testlib.mk - # ----------------------------------------------------------------------------- # Libraries used by dlfcn tests to verify correct load order: # libtest_check_order_2_right.so diff --git a/tests/libs/dlopen_nodelete_1.cpp b/tests/libs/dlopen_nodelete_1.cpp deleted file mode 100644 index 943897815..000000000 --- a/tests/libs/dlopen_nodelete_1.cpp +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -uint32_t dlopen_nodelete_1_taxicab_number = 1729; -static bool* unload_flag_ptr = nullptr; - -extern "C" void dlopen_nodelete_1_set_unload_flag_ptr(bool* ptr) { - unload_flag_ptr = ptr; -} - -static void __attribute__((destructor)) unload_guard() { - if (unload_flag_ptr != nullptr) { - *unload_flag_ptr = true; - } -} diff --git a/tests/libs/dlopen_nodelete_2.cpp b/tests/libs/dlopen_nodelete_2.cpp deleted file mode 100644 index b5ab5c1ae..000000000 --- a/tests/libs/dlopen_nodelete_2.cpp +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -uint32_t dlopen_nodelete_2_taxicab_number = 1729; -static bool* unload_flag_ptr = nullptr; - -extern "C" void dlopen_nodelete_2_set_unload_flag_ptr(bool* ptr) { - unload_flag_ptr = ptr; -} - -static void __attribute__((destructor)) unload_guard() { - if (unload_flag_ptr != nullptr) { - *unload_flag_ptr = true; - } -} diff --git a/tests/libs/dlopen_nodelete_dt_flags_1.cpp b/tests/libs/dlopen_nodelete_dt_flags_1.cpp deleted file mode 100644 index 39c0a7ea6..000000000 --- a/tests/libs/dlopen_nodelete_dt_flags_1.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -static bool* unload_flag_ptr = nullptr; - -extern "C" void dlopen_nodelete_dt_flags_1_set_unload_flag_ptr(bool* ptr) { - unload_flag_ptr = ptr; -} - -static void __attribute__((destructor)) unload_guard() { - if (unload_flag_ptr != nullptr) { - *unload_flag_ptr = true; - } -} From e4ae96ffd3bf23b510c53701b14555cfd0274499 Mon Sep 17 00:00:00 2001 From: Dmitriy Ivanov Date: Mon, 3 Nov 2014 22:12:19 -0800 Subject: [PATCH 22/89] Revert "Fix dlsym() to take into account RTLD_GLOBAL/LOCAL" This reverts commit c85e82dde5c4b2accc50a9e17740b9005dfbae6a. Bug: 18222321 Bug: 18211780 Change-Id: I32f4048bd5ea85dc8a3dfccce8cf141b241ab692 --- linker/dlfcn.cpp | 2 +- linker/linker.cpp | 48 ++++++++++------------------ linker/linker.h | 4 +-- tests/dlfcn_test.cpp | 36 --------------------- tests/libs/dlopen_testlib_simple.cpp | 2 +- 5 files changed, 20 insertions(+), 72 deletions(-) diff --git a/linker/dlfcn.cpp b/linker/dlfcn.cpp index 367179d8d..3eb5bea22 100644 --- a/linker/dlfcn.cpp +++ b/linker/dlfcn.cpp @@ -232,7 +232,7 @@ static unsigned g_libdl_chains[] = { 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0 }; static unsigned g_libdl_chains[] = { 0, 2, 3, 4, 5, 6, 7, 8, 9, 0 }; #endif -static soinfo __libdl_info("libdl.so", nullptr, 0, RTLD_GLOBAL); +static soinfo __libdl_info("libdl.so", nullptr, 0); // This is used by the dynamic linker. Every process gets these symbols for free. soinfo* get_libdl_info() { diff --git a/linker/linker.cpp b/linker/linker.cpp index 636541297..35c8cbdc8 100644 --- a/linker/linker.cpp +++ b/linker/linker.cpp @@ -282,13 +282,13 @@ static void protect_data(int protection) { g_soinfo_links_allocator.protect_all(protection); } -static soinfo* soinfo_alloc(const char* name, struct stat* file_stat, off64_t file_offset, int rtld_flags) { +static soinfo* soinfo_alloc(const char* name, struct stat* file_stat, off64_t file_offset) { if (strlen(name) >= SOINFO_NAME_LEN) { DL_ERR("library name \"%s\" too long", name); return nullptr; } - soinfo* si = new (g_soinfo_allocator.alloc()) soinfo(name, file_stat, file_offset, rtld_flags); + soinfo* si = new (g_soinfo_allocator.alloc()) soinfo(name, file_stat, file_offset); sonext->next = si; sonext = si; @@ -452,7 +452,7 @@ static ElfW(Sym)* soinfo_elf_lookup(soinfo* si, unsigned hash, const char* name) return nullptr; } -soinfo::soinfo(const char* name, const struct stat* file_stat, off64_t file_offset, int rtld_flags) { +soinfo::soinfo(const char* name, const struct stat* file_stat, off64_t file_offset) { memset(this, 0, sizeof(*this)); strlcpy(this->name, name, sizeof(this->name)); @@ -464,8 +464,6 @@ soinfo::soinfo(const char* name, const struct stat* file_stat, off64_t file_offs this->st_ino = file_stat->st_ino; this->file_offset = file_offset; } - - this->rtld_flags = rtld_flags; } static unsigned elfhash(const char* _name) { @@ -686,10 +684,6 @@ ElfW(Sym)* dlsym_linear_lookup(const char* name, soinfo** found, soinfo* start) ElfW(Sym)* s = nullptr; for (soinfo* si = start; (s == nullptr) && (si != nullptr); si = si->next) { - if ((si->get_rtld_flags() & RTLD_GLOBAL) == 0) { - continue; - } - s = soinfo_elf_lookup(si, elf_hash, name); if (s != nullptr) { *found = si; @@ -780,7 +774,7 @@ static void for_each_dt_needed(const soinfo* si, F action) { } } -static soinfo* load_library(LoadTaskList& load_tasks, const char* name, int rtld_flags, const android_dlextinfo* extinfo) { +static soinfo* load_library(LoadTaskList& load_tasks, const char* name, int dlflags, const android_dlextinfo* extinfo) { int fd = -1; off64_t file_offset = 0; ScopedFd file_guard(-1); @@ -825,7 +819,7 @@ static soinfo* load_library(LoadTaskList& load_tasks, const char* name, int rtld } } - if ((rtld_flags & RTLD_NOLOAD) != 0) { + if ((dlflags & RTLD_NOLOAD) != 0) { DL_ERR("library \"%s\" wasn't loaded and RTLD_NOLOAD prevented it", name); return nullptr; } @@ -836,7 +830,7 @@ static soinfo* load_library(LoadTaskList& load_tasks, const char* name, int rtld return nullptr; } - soinfo* si = soinfo_alloc(SEARCH_NAME(name), &file_stat, file_offset, rtld_flags); + soinfo* si = soinfo_alloc(SEARCH_NAME(name), &file_stat, file_offset); if (si == nullptr) { return nullptr; } @@ -868,7 +862,7 @@ static soinfo *find_loaded_library_by_name(const char* name) { return nullptr; } -static soinfo* find_library_internal(LoadTaskList& load_tasks, const char* name, int rtld_flags, const android_dlextinfo* extinfo) { +static soinfo* find_library_internal(LoadTaskList& load_tasks, const char* name, int dlflags, const android_dlextinfo* extinfo) { soinfo* si = find_loaded_library_by_name(name); @@ -876,7 +870,7 @@ static soinfo* find_library_internal(LoadTaskList& load_tasks, const char* name, // of this fact is done by load_library. if (si == nullptr) { TRACE("[ '%s' has not been found by name. Trying harder...]", name); - si = load_library(load_tasks, name, rtld_flags, extinfo); + si = load_library(load_tasks, name, dlflags, extinfo); } return si; @@ -900,7 +894,7 @@ static bool is_recursive(soinfo* si, soinfo* parent) { } static bool find_libraries(const char* const library_names[], size_t library_names_size, soinfo* soinfos[], - soinfo* ld_preloads[], size_t ld_preloads_size, int rtld_flags, const android_dlextinfo* extinfo) { + soinfo* ld_preloads[], size_t ld_preloads_size, int dlflags, const android_dlextinfo* extinfo) { // Step 0: prepare. LoadTaskList load_tasks; for (size_t i = 0; i < library_names_size; ++i) { @@ -926,7 +920,7 @@ static bool find_libraries(const char* const library_names[], size_t library_nam // Step 1: load and pre-link all DT_NEEDED libraries in breadth first order. for (LoadTask::unique_ptr task(load_tasks.pop_front()); task.get() != nullptr; task.reset(load_tasks.pop_front())) { - soinfo* si = find_library_internal(load_tasks, task->get_name(), rtld_flags, extinfo); + soinfo* si = find_library_internal(load_tasks, task->get_name(), dlflags, extinfo); if (si == nullptr) { return false; } @@ -971,7 +965,7 @@ static bool find_libraries(const char* const library_names[], size_t library_nam return true; } -static soinfo* find_library(const char* name, int rtld_flags, const android_dlextinfo* extinfo) { +static soinfo* find_library(const char* name, int dlflags, const android_dlextinfo* extinfo) { if (name == nullptr) { somain->ref_count++; return somain; @@ -979,7 +973,7 @@ static soinfo* find_library(const char* name, int rtld_flags, const android_dlex soinfo* si; - if (!find_libraries(&name, 1, &si, nullptr, 0, rtld_flags, extinfo)) { + if (!find_libraries(&name, 1, &si, nullptr, 0, dlflags, extinfo)) { return nullptr; } @@ -1764,14 +1758,6 @@ off64_t soinfo::get_file_offset() { return 0; } -int soinfo::get_rtld_flags() { - if (has_min_version(1)) { - return rtld_flags; - } - - return 0; -} - // This is a return on get_children()/get_parents() if // 'this->flags' does not have FLAG_NEW_SOINFO set. static soinfo::soinfo_list_t g_empty_list; @@ -2289,7 +2275,7 @@ static void add_vdso(KernelArgumentBlock& args __unused) { return; } - soinfo* si = soinfo_alloc("[vdso]", nullptr, 0, 0); + soinfo* si = soinfo_alloc("[vdso]", nullptr, 0); si->phdr = reinterpret_cast(reinterpret_cast(ehdr_vdso) + ehdr_vdso->e_phoff); si->phnum = ehdr_vdso->e_phnum; @@ -2310,7 +2296,7 @@ static void add_vdso(KernelArgumentBlock& args __unused) { #else #define LINKER_PATH "/system/bin/linker" #endif -static soinfo linker_soinfo_for_gdb(LINKER_PATH, nullptr, 0, 0); +static soinfo linker_soinfo_for_gdb(LINKER_PATH, nullptr, 0); /* gdb expects the linker to be in the debug shared object list. * Without this, gdb has trouble locating the linker's ".text" @@ -2374,7 +2360,7 @@ static ElfW(Addr) __linker_init_post_relocation(KernelArgumentBlock& args, ElfW( INFO("[ android linker & debugger ]"); - soinfo* si = soinfo_alloc(args.argv[0], nullptr, 0, RTLD_GLOBAL); + soinfo* si = soinfo_alloc(args.argv[0], nullptr, 0); if (si == nullptr) { exit(EXIT_FAILURE); } @@ -2449,7 +2435,7 @@ static ElfW(Addr) __linker_init_post_relocation(KernelArgumentBlock& args, ElfW( memset(needed_library_names, 0, sizeof(needed_library_names)); needed_library_name_list.copy_to_array(needed_library_names, needed_libraries_count); - if (needed_libraries_count > 0 && !find_libraries(needed_library_names, needed_libraries_count, needed_library_si, g_ld_preloads, ld_preloads_count, RTLD_GLOBAL, nullptr)) { + if (needed_libraries_count > 0 && !find_libraries(needed_library_names, needed_libraries_count, needed_library_si, g_ld_preloads, ld_preloads_count, 0, nullptr)) { __libc_format_fd(2, "CANNOT LINK EXECUTABLE DEPENDENCIES: %s\n", linker_get_error_buffer()); exit(EXIT_FAILURE); } @@ -2562,7 +2548,7 @@ extern "C" ElfW(Addr) __linker_init(void* raw_args) { ElfW(Ehdr)* elf_hdr = reinterpret_cast(linker_addr); ElfW(Phdr)* phdr = reinterpret_cast(linker_addr + elf_hdr->e_phoff); - soinfo linker_so("[dynamic linker]", nullptr, 0, 0); + soinfo linker_so("[dynamic linker]", nullptr, 0); // If the linker is not acting as PT_INTERP entry_point is equal to // _start. Which means that the linker is running as an executable and diff --git a/linker/linker.h b/linker/linker.h index 6329efda6..fa38c7fef 100644 --- a/linker/linker.h +++ b/linker/linker.h @@ -201,7 +201,7 @@ struct soinfo { #endif bool has_DT_SYMBOLIC; - soinfo(const char* name, const struct stat* file_stat, off64_t file_offset, int rtld_flags); + soinfo(const char* name, const struct stat* file_stat, off64_t file_offset); void CallConstructors(); void CallDestructors(); @@ -216,8 +216,6 @@ struct soinfo { dev_t get_st_dev(); off64_t get_file_offset(); - int get_rtld_flags(); - soinfo_list_t& get_children(); soinfo_list_t& get_parents(); diff --git a/tests/dlfcn_test.cpp b/tests/dlfcn_test.cpp index c9c856a37..e24af13c0 100644 --- a/tests/dlfcn_test.cpp +++ b/tests/dlfcn_test.cpp @@ -202,42 +202,6 @@ TEST(dlfcn, dlopen_check_order) { dlclose(handle); } -TEST(dlfcn, dlopen_check_rtld_local) { - void* sym = dlsym(RTLD_DEFAULT, "dlopen_testlib_simple_func"); - ASSERT_TRUE(sym == nullptr); - - // implicit RTLD_LOCAL - void* handle = dlopen("libtest_simple.so", RTLD_NOW); - sym = dlsym(RTLD_DEFAULT, "dlopen_testlib_simple_func"); - ASSERT_TRUE(sym == nullptr); - ASSERT_SUBSTR("undefined symbol: dlopen_testlib_simple_func", dlerror()); - sym = dlsym(handle, "dlopen_testlib_simple_func"); - ASSERT_TRUE(sym != nullptr); - ASSERT_TRUE(reinterpret_cast(sym)()); - dlclose(handle); - - // explicit RTLD_LOCAL - handle = dlopen("libtest_simple.so", RTLD_NOW | RTLD_LOCAL); - sym = dlsym(RTLD_DEFAULT, "dlopen_testlib_simple_func"); - ASSERT_TRUE(sym == nullptr); - ASSERT_SUBSTR("undefined symbol: dlopen_testlib_simple_func", dlerror()); - sym = dlsym(handle, "dlopen_testlib_simple_func"); - ASSERT_TRUE(sym != nullptr); - ASSERT_TRUE(reinterpret_cast(sym)()); - dlclose(handle); -} - -TEST(dlfcn, dlopen_check_rtld_global) { - void* sym = dlsym(RTLD_DEFAULT, "dlopen_testlib_simple_func"); - ASSERT_TRUE(sym == nullptr); - - void* handle = dlopen("libtest_simple.so", RTLD_NOW | RTLD_GLOBAL); - sym = dlsym(RTLD_DEFAULT, "dlopen_testlib_simple_func"); - ASSERT_TRUE(sym != nullptr) << dlerror(); - ASSERT_TRUE(reinterpret_cast(sym)()); - dlclose(handle); -} - // libtest_with_dependency_loop.so -> libtest_with_dependency_loop_a.so -> // libtest_with_dependency_loop_b.so -> libtest_with_dependency_loop_c.so -> // libtest_with_dependency_loop_a.so diff --git a/tests/libs/dlopen_testlib_simple.cpp b/tests/libs/dlopen_testlib_simple.cpp index 32269557a..bf750b2a6 100644 --- a/tests/libs/dlopen_testlib_simple.cpp +++ b/tests/libs/dlopen_testlib_simple.cpp @@ -19,6 +19,6 @@ uint32_t dlopen_testlib_taxicab_number = 1729; -extern "C" bool dlopen_testlib_simple_func() { +bool dlopen_testlib_simple_func() { return true; } From bcc61c8c04ff7e76cc9c64878f65ecd43cb0ad50 Mon Sep 17 00:00:00 2001 From: Manish Kumar Date: Mon, 9 Feb 2015 19:21:46 +0530 Subject: [PATCH 23/89] Remove duplicate definition of stpcpy. Change-Id: Ie11e9e43d7179c13feed48001a83aed310fe6e3e --- libc/arch-arm/cortex-a15/cortex-a15.mk | 1 - 1 file changed, 1 deletion(-) diff --git a/libc/arch-arm/cortex-a15/cortex-a15.mk b/libc/arch-arm/cortex-a15/cortex-a15.mk index f1abe3231..f3c55018d 100644 --- a/libc/arch-arm/cortex-a15/cortex-a15.mk +++ b/libc/arch-arm/cortex-a15/cortex-a15.mk @@ -1,7 +1,6 @@ libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/memcpy.S \ arch-arm/cortex-a15/bionic/memset.S \ - arch-arm/cortex-a15/bionic/stpcpy.S \ arch-arm/cortex-a15/bionic/strcat.S \ arch-arm/cortex-a15/bionic/__strcat_chk.S \ arch-arm/cortex-a15/bionic/strcmp.S \ From 906f26eede022836ce804b404b8078be7070b001 Mon Sep 17 00:00:00 2001 From: AUDITYA BHATTARAM Date: Thu, 12 Mar 2015 13:37:11 +0530 Subject: [PATCH 24/89] Remove duplicate definition of stpcpy.c for emulator. Remove duplicate definition of stpcpy.c for emulator. Change-Id: I359354330729f7420bef8bca01cdecf43b58b535 --- libc/arch-arm/generic/generic.mk | 1 - 1 file changed, 1 deletion(-) diff --git a/libc/arch-arm/generic/generic.mk b/libc/arch-arm/generic/generic.mk index 95be867cd..2456e6e4c 100644 --- a/libc/arch-arm/generic/generic.mk +++ b/libc/arch-arm/generic/generic.mk @@ -7,5 +7,4 @@ libc_bionic_src_files_arm += \ bionic/memmove.c \ bionic/__strcat_chk.cpp \ bionic/__strcpy_chk.cpp \ - upstream-openbsd/lib/libc/string/stpcpy.c \ upstream-openbsd/lib/libc/string/strcat.c \ From 193c39ce03b9897397fe214f948c1fdbd6fa9899 Mon Sep 17 00:00:00 2001 From: Ricardo Cerqueira Date: Sun, 9 Nov 2014 00:14:07 +0000 Subject: [PATCH 25/89] Partial revert "Remove ioprio_get(2) and ioprio_set(2) from LP64." Put them back for ARM32. Some pre-L blobs link with it This partially reverts commit 607341e226912d95d03216483bdcef6f8d96f8b4. Change-Id: If8503bd5624869ba071fdd21fa48cffb65617586 --- libc/arch-arm/syscalls/ioprio_get.S | 14 ++++++++++++++ libc/arch-arm/syscalls/ioprio_set.S | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 libc/arch-arm/syscalls/ioprio_get.S create mode 100644 libc/arch-arm/syscalls/ioprio_set.S diff --git a/libc/arch-arm/syscalls/ioprio_get.S b/libc/arch-arm/syscalls/ioprio_get.S new file mode 100644 index 000000000..19e04ed1f --- /dev/null +++ b/libc/arch-arm/syscalls/ioprio_get.S @@ -0,0 +1,14 @@ +/* Generated by gensyscalls.py. Do not edit. */ + +#include + +ENTRY(ioprio_get) + mov ip, r7 + ldr r7, =__NR_ioprio_get + swi #0 + mov r7, ip + cmn r0, #(MAX_ERRNO + 1) + bxls lr + neg r0, r0 + b __set_errno +END(ioprio_get) diff --git a/libc/arch-arm/syscalls/ioprio_set.S b/libc/arch-arm/syscalls/ioprio_set.S new file mode 100644 index 000000000..9b94efb59 --- /dev/null +++ b/libc/arch-arm/syscalls/ioprio_set.S @@ -0,0 +1,14 @@ +/* Generated by gensyscalls.py. Do not edit. */ + +#include + +ENTRY(ioprio_set) + mov ip, r7 + ldr r7, =__NR_ioprio_set + swi #0 + mov r7, ip + cmn r0, #(MAX_ERRNO + 1) + bxls lr + neg r0, r0 + b __set_errno +END(ioprio_set) From aa729cb8dd274ea5a6bfc6055a035780478b2d68 Mon Sep 17 00:00:00 2001 From: Flemmard Date: Thu, 13 Nov 2014 00:12:22 +0100 Subject: [PATCH 26/89] bionic: Add flag to restore legacy mmap behavior * Pre-lollipop mmap would not care whether offset was signed or unsigned. * Lollipop adds 64-bit support which results in sign extension of offset, causing a negative offset when a positive offset > 2^31 is given. Change-Id: I5d19d898fc131cf848217974915d1b466a474f99 --- libc/Android.mk | 4 ++++ libc/bionic/mmap.cpp | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/libc/Android.mk b/libc/Android.mk index 30050922b..8e1e2e462 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -535,6 +535,10 @@ ifneq ($(BOARD_MALLOC_ALIGNMENT),) libc_common_cflags += -DMALLOC_ALIGNMENT=$(BOARD_MALLOC_ALIGNMENT) endif +ifeq ($(BOARD_USES_LEGACY_MMAP),true) + libc_common_cflags += -DLEGACY_MMAP +endif + # Define ANDROID_SMP appropriately. ifeq ($(TARGET_CPU_SMP),true) libc_common_cflags += -DANDROID_SMP=1 diff --git a/libc/bionic/mmap.cpp b/libc/bionic/mmap.cpp index 8f25a8941..53e8b4615 100644 --- a/libc/bionic/mmap.cpp +++ b/libc/bionic/mmap.cpp @@ -36,6 +36,11 @@ extern "C" void* __mmap2(void*, size_t, int, int, int, size_t); #define MMAP2_SHIFT 12 // 2**12 == 4096 +#ifdef LEGACY_MMAP +#define TO_64(a) ((a) & 0x00000000ffffffff) +#else +#define TO_64(a) (a) +#endif static bool kernel_has_MADV_MERGEABLE = true; @@ -60,5 +65,5 @@ void* mmap64(void* addr, size_t size, int prot, int flags, int fd, off64_t offse } void* mmap(void* addr, size_t size, int prot, int flags, int fd, off_t offset) { - return mmap64(addr, size, prot, flags, fd, static_cast(offset)); + return mmap64(addr, size, prot, flags, fd, TO_64(static_cast(offset))); } From a12722efa7c4238e38475dea588e080f23e08b53 Mon Sep 17 00:00:00 2001 From: Bernhard Rosenkraenzer Date: Sat, 15 Feb 2014 20:43:47 +0100 Subject: [PATCH 27/89] Replace Cortex A15 strlen implementation with cortex-strings Benchmarks on a Nexus 10 have shown this implementation to be 5%-20% faster (depending on input) Change-Id: I4331a390c404a77b1e13909d713c4258a3776bca Signed-off-by: Bernhard Rosenkraenzer --- libc/arch-arm/cortex-a15/bionic/strlen.S | 293 +++++++++++------------ 1 file changed, 137 insertions(+), 156 deletions(-) diff --git a/libc/arch-arm/cortex-a15/bionic/strlen.S b/libc/arch-arm/cortex-a15/bionic/strlen.S index 9a0ce62a9..8545132fb 100644 --- a/libc/arch-arm/cortex-a15/bionic/strlen.S +++ b/libc/arch-arm/cortex-a15/bionic/strlen.S @@ -1,165 +1,146 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. +/* Copyright (c) 2010-2011,2013 Linaro Limited + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Linaro Limited nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + /* - * Copyright (c) 2013 ARM Ltd - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the company may not be used to endorse or promote - * products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + Assumes: + ARMv6T2, AArch32 + + Adapted to Bionic by Bernhard Rosenkränzer */ #include - .syntax unified - - .thumb - .thumb_func +#ifdef __ARMEB__ +#define S2LO lsl +#define S2HI lsr +#else +#define S2LO lsr +#define S2HI lsl +#endif + + .text + /* This code requires Thumb. */ + .thumb + .syntax unified + +/* Parameters and result. */ +#define srcin r0 +#define result r0 + +/* Internal variables. */ +#define src r1 +#define data1a r2 +#define data1b r3 +#define const_m1 r12 +#define const_0 r4 +#define tmp1 r4 /* Overlaps const_0 */ +#define tmp2 r5 ENTRY(strlen) - pld [r0, #0] - mov r1, r0 - - ands r3, r0, #7 - beq mainloop - - // Align to a double word (64 bits). - rsb r3, r3, #8 - lsls ip, r3, #31 - beq align_to_32 - - ldrb r2, [r1], #1 - cbz r2, update_count_and_return - -align_to_32: - bcc align_to_64 - ands ip, r3, #2 - beq align_to_64 - - ldrb r2, [r1], #1 - cbz r2, update_count_and_return - ldrb r2, [r1], #1 - cbz r2, update_count_and_return - -align_to_64: - tst r3, #4 - beq mainloop - ldr r3, [r1], #4 - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne zero_in_second_register - - .p2align 2 -mainloop: - ldrd r2, r3, [r1], #8 - - pld [r1, #64] - - sub ip, r2, #0x01010101 - bic ip, ip, r2 - ands ip, ip, #0x80808080 - bne zero_in_first_register - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne zero_in_second_register - b mainloop - -update_count_and_return: - sub r0, r1, r0 - sub r0, r0, #1 - bx lr - -zero_in_first_register: - sub r0, r1, r0 - lsls r3, ip, #17 - bne sub8_and_return - bcs sub7_and_return - lsls ip, ip, #1 - bne sub6_and_return - - sub r0, r0, #5 - bx lr - -sub8_and_return: - sub r0, r0, #8 - bx lr - -sub7_and_return: - sub r0, r0, #7 - bx lr - -sub6_and_return: - sub r0, r0, #6 - bx lr - -zero_in_second_register: - sub r0, r1, r0 - lsls r3, ip, #17 - bne sub4_and_return - bcs sub3_and_return - lsls ip, ip, #1 - bne sub2_and_return - - sub r0, r0, #1 - bx lr - -sub4_and_return: - sub r0, r0, #4 - bx lr - -sub3_and_return: - sub r0, r0, #3 - bx lr - -sub2_and_return: - sub r0, r0, #2 - bx lr + .p2align 6 + pld [srcin, #0] + strd r4, r5, [sp, #-8]! + bic src, srcin, #7 + mvn const_m1, #0 + ands tmp1, srcin, #7 /* (8 - bytes) to alignment. */ + pld [src, #32] + bne.w misaligned8 + mov const_0, #0 + mov result, #-8 +loop_aligned: + /* Bytes 0-7. */ + ldrd data1a, data1b, [src] + pld [src, #64] + add result, result, #8 +start_realigned: + uadd8 data1a, data1a, const_m1 /* Saturating GE<0:3> set. */ + sel data1a, const_0, const_m1 /* Select based on GE<0:3>. */ + uadd8 data1b, data1b, const_m1 + sel data1b, data1a, const_m1 /* Only used if d1a == 0. */ + cbnz data1b, null_found + + /* Bytes 8-15. */ + ldrd data1a, data1b, [src, #8] + uadd8 data1a, data1a, const_m1 /* Saturating GE<0:3> set. */ + add result, result, #8 + sel data1a, const_0, const_m1 /* Select based on GE<0:3>. */ + uadd8 data1b, data1b, const_m1 + sel data1b, data1a, const_m1 /* Only used if d1a == 0. */ + cbnz data1b, null_found + + /* Bytes 16-23. */ + ldrd data1a, data1b, [src, #16] + uadd8 data1a, data1a, const_m1 /* Saturating GE<0:3> set. */ + add result, result, #8 + sel data1a, const_0, const_m1 /* Select based on GE<0:3>. */ + uadd8 data1b, data1b, const_m1 + sel data1b, data1a, const_m1 /* Only used if d1a == 0. */ + cbnz data1b, null_found + + /* Bytes 24-31. */ + ldrd data1a, data1b, [src, #24] + add src, src, #32 + uadd8 data1a, data1a, const_m1 /* Saturating GE<0:3> set. */ + add result, result, #8 + sel data1a, const_0, const_m1 /* Select based on GE<0:3>. */ + uadd8 data1b, data1b, const_m1 + sel data1b, data1a, const_m1 /* Only used if d1a == 0. */ + cmp data1b, #0 + beq loop_aligned + +null_found: + cmp data1a, #0 + itt eq + addeq result, result, #4 + moveq data1a, data1b +#ifndef __ARMEB__ + rev data1a, data1a +#endif + clz data1a, data1a + ldrd r4, r5, [sp], #8 + add result, result, data1a, lsr #3 /* Bits -> Bytes. */ + bx lr + +misaligned8: + ldrd data1a, data1b, [src] + and tmp2, tmp1, #3 + rsb result, tmp1, #0 + lsl tmp2, tmp2, #3 /* Bytes -> bits. */ + tst tmp1, #4 + pld [src, #64] + S2HI tmp2, const_m1, tmp2 + orn data1a, data1a, tmp2 + itt ne + ornne data1b, data1b, tmp2 + movne data1a, const_m1 + mov const_0, #0 + b start_realigned END(strlen) From 9f7adb469946bb4375f9f4489517221702529740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= Date: Thu, 27 Feb 2014 01:30:07 +0100 Subject: [PATCH 28/89] Import memchr implementation from cortex-strings for Cortex A15 Benchmarks on a Nexus 10 have shown the cortex-strings implementation of memchr to be 60%-70% faster (depending on input) Change-Id: I525587d8663c881e9059b26b19409b0c563b2a18 Signed-off-by: Bernhard Rosenkraenzer --- libc/arch-arm/arm.mk | 1 - libc/arch-arm/cortex-a15/bionic/memchr.S | 153 +++++++++++++++++++++++ libc/arch-arm/cortex-a15/cortex-a15.mk | 1 + libc/arch-arm/cortex-a9/cortex-a9.mk | 1 + libc/arch-arm/generic/generic.mk | 1 + libc/arch-arm/krait/krait.mk | 1 + 6 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 libc/arch-arm/cortex-a15/bionic/memchr.S diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index de4f6eb8f..d4ca00783 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -23,7 +23,6 @@ libc_openbsd_src_files_arm += \ # libc_bionic_src_files_arm += \ - bionic/memchr.c \ bionic/memrchr.c \ bionic/strchr.cpp \ bionic/strnlen.c \ diff --git a/libc/arch-arm/cortex-a15/bionic/memchr.S b/libc/arch-arm/cortex-a15/bionic/memchr.S new file mode 100644 index 000000000..784d44a4c --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/memchr.S @@ -0,0 +1,153 @@ +/* Copyright (c) 2010-2011, Linaro Limited + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Linaro Limited nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + Written by Dave Gilbert + Adapted to work inside Bionic by Bernhard Rosenkränzer + + This memchr routine is optimised on a Cortex-A9 and should work on + all ARMv7 processors. It has a fast past for short sizes, and has + an optimised path for large data sets; the worst case is finding the + match early in a large data set. + + */ + +// Prototype: void *memchr(const void *s, int c, size_t n); + +#include +#include "private/libc_events.h" + + .syntax unified + .arch armv7-a + +@ this lets us check a flag in a 00/ff byte easily in either endianness +#ifdef __ARMEB__ +#define CHARTSTMASK(c) 1<<(31-(c*8)) +#else +#define CHARTSTMASK(c) 1<<(c*8) +#endif + .text + .thumb + +@ --------------------------------------------------------------------------- +ENTRY(memchr) + .thumb_func + .align 2 + .p2align 4,,15 + @ r0 = start of memory to scan + @ r1 = character to look for + @ r2 = length + @ returns r0 = pointer to character or NULL if not found + and r1,r1,#0xff @ Don't think we can trust the caller to actually pass a char + + cmp r2,#16 @ If it's short don't bother with anything clever + blt 20f + + tst r0, #7 @ If it's already aligned skip the next bit + beq 10f + + @ Work up to an aligned point +5: + ldrb r3, [r0],#1 + subs r2, r2, #1 + cmp r3, r1 + beq 50f @ If it matches exit found + tst r0, #7 + bne 5b @ If not aligned yet then do next byte + +10: + @ At this point, we are aligned, we know we have at least 8 bytes to work with + push {r4,r5,r6,r7} + orr r1, r1, r1, lsl #8 @ expand the match word across to all bytes + orr r1, r1, r1, lsl #16 + bic r4, r2, #7 @ Number of double words to work with + mvns r7, #0 @ all F's + movs r3, #0 + +15: + ldmia r0!,{r5,r6} + subs r4, r4, #8 + eor r5,r5, r1 @ Get it so that r5,r6 have 00's where the bytes match the target + eor r6,r6, r1 + uadd8 r5, r5, r7 @ Parallel add 0xff - sets the GE bits for anything that wasn't 0 + sel r5, r3, r7 @ bytes are 00 for none-00 bytes, or ff for 00 bytes - NOTE INVERSION + uadd8 r6, r6, r7 @ Parallel add 0xff - sets the GE bits for anything that wasn't 0 + sel r6, r5, r7 @ chained....bytes are 00 for none-00 bytes, or ff for 00 bytes - NOTE INVERSION + cbnz r6, 60f + bne 15b @ (Flags from the subs above) If not run out of bytes then go around again + + pop {r4,r5,r6,r7} + and r1,r1,#0xff @ Get r1 back to a single character from the expansion above + and r2,r2,#7 @ Leave the count remaining as the number after the double words have been done + +20: + cbz r2, 40f @ 0 length or hit the end already then not found + +21: @ Post aligned section, or just a short call + ldrb r3,[r0],#1 + subs r2,r2,#1 + eor r3,r3,r1 @ r3 = 0 if match - doesn't break flags from sub + cbz r3, 50f + bne 21b @ on r2 flags + +40: + movs r0,#0 @ not found + bx lr + +50: + subs r0,r0,#1 @ found + bx lr + +60: @ We're here because the fast path found a hit - now we have to track down exactly which word it was + @ r0 points to the start of the double word after the one that was tested + @ r5 has the 00/ff pattern for the first word, r6 has the chained value + cmp r5, #0 + itte eq + moveq r5, r6 @ the end is in the 2nd word + subeq r0,r0,#3 @ Points to 2nd byte of 2nd word + subne r0,r0,#7 @ or 2nd byte of 1st word + + @ r0 currently points to the 3rd byte of the word containing the hit + tst r5, # CHARTSTMASK(0) @ 1st character + bne 61f + adds r0,r0,#1 + tst r5, # CHARTSTMASK(1) @ 2nd character + ittt eq + addeq r0,r0,#1 + tsteq r5, # (3<<15) @ 2nd & 3rd character + @ If not the 3rd must be the last one + addeq r0,r0,#1 + +61: + pop {r4,r5,r6,r7} + subs r0,r0,#1 + bx lr +END(memchr) diff --git a/libc/arch-arm/cortex-a15/cortex-a15.mk b/libc/arch-arm/cortex-a15/cortex-a15.mk index f3c55018d..ac5fbbe57 100644 --- a/libc/arch-arm/cortex-a15/cortex-a15.mk +++ b/libc/arch-arm/cortex-a15/cortex-a15.mk @@ -1,4 +1,5 @@ libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memchr.S \ arch-arm/cortex-a15/bionic/memcpy.S \ arch-arm/cortex-a15/bionic/memset.S \ arch-arm/cortex-a15/bionic/strcat.S \ diff --git a/libc/arch-arm/cortex-a9/cortex-a9.mk b/libc/arch-arm/cortex-a9/cortex-a9.mk index c82db3b4d..5805ea86c 100644 --- a/libc/arch-arm/cortex-a9/cortex-a9.mk +++ b/libc/arch-arm/cortex-a9/cortex-a9.mk @@ -1,4 +1,5 @@ libc_bionic_src_files_arm += \ + bionic/memchr.c \ arch-arm/cortex-a9/bionic/memcpy.S \ arch-arm/cortex-a9/bionic/memset.S \ arch-arm/cortex-a9/bionic/stpcpy.S \ diff --git a/libc/arch-arm/generic/generic.mk b/libc/arch-arm/generic/generic.mk index 2456e6e4c..760c5bc78 100644 --- a/libc/arch-arm/generic/generic.mk +++ b/libc/arch-arm/generic/generic.mk @@ -1,4 +1,5 @@ libc_bionic_src_files_arm += \ + bionic/memchr.c \ arch-arm/generic/bionic/memcpy.S \ arch-arm/generic/bionic/memset.S \ arch-arm/generic/bionic/strcmp.S \ diff --git a/libc/arch-arm/krait/krait.mk b/libc/arch-arm/krait/krait.mk index 67232f0d2..5947dcb7b 100644 --- a/libc/arch-arm/krait/krait.mk +++ b/libc/arch-arm/krait/krait.mk @@ -1,4 +1,5 @@ libc_bionic_src_files_arm += \ + bionic/memchr.c \ arch-arm/krait/bionic/memcpy.S \ arch-arm/krait/bionic/memset.S \ arch-arm/krait/bionic/strcmp.S \ From 554721b671b5e9216aaa0d7603fadaca1c592b3e Mon Sep 17 00:00:00 2001 From: Bernhard Rosenkraenzer Date: Sat, 15 Feb 2014 21:10:20 +0100 Subject: [PATCH 29/89] Replace memcpy with the cortex-strings implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Benchmarks on a Nexus 10 have shown this implementation to be up to 12% faster. Change-Id: Ida882cc6a583dfd00c3659b022c26070665824af Signed-off-by: Bernhard Rosenkränzer --- libc/arch-arm/cortex-a15/bionic/memcpy.S | 669 ++++++++++++++++++++--- 1 file changed, 606 insertions(+), 63 deletions(-) diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy.S b/libc/arch-arm/cortex-a15/bionic/memcpy.S index 410b663e2..5591d78cd 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy.S @@ -1,80 +1,623 @@ -/* - * Copyright (C) 2008 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. +/* Copyright (c) 2013, Linaro Limited + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Linaro Limited nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + /* - * Copyright (c) 2013 ARM Ltd - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the company may not be used to endorse or promote - * products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ + This memcpy routine is optimised for Cortex-A15 cores and takes advantage + of VFP or NEON when built with the appropriate flags. + + Assumptions: -// Prototype: void *memcpy (void *dst, const void *src, size_t count). + ARMv6 (ARMv7-a if using Neon) + ARM state + Unaligned accesses + + Ported to work inside Bionic by Bernhard Rosenkränzer + */ #include #include - .text - .syntax unified - .fpu neon + .syntax unified + /* This implementation requires ARM state. */ + .arm + +#ifdef __ARM_NEON__ + + .fpu neon + .arch armv7-a +# define FRAME_SIZE 4 +# define USE_VFP +# define USE_NEON + +#elif !defined (__SOFTFP__) +#error VFP + + .arch armv6 + .fpu vfpv2 +# define FRAME_SIZE 32 +# define USE_VFP + +#else +#error NO FPU + .arch armv6 +# define FRAME_SIZE 32 + +#endif + +/* Old versions of GAS incorrectly implement the NEON align semantics. */ +#ifdef BROKEN_ASM_NEON_ALIGN +#define ALIGN(addr, align) addr,:align +#else +#define ALIGN(addr, align) addr:align +#endif + +#define PC_OFFSET 8 /* PC pipeline compensation. */ +#define INSN_SIZE 4 + +/* Call parameters. */ +#define dstin r0 +#define src r1 +#define count r2 + +/* Locals. */ +#define tmp1 r3 +#define dst ip +#define tmp2 r10 + +#ifndef USE_NEON +/* For bulk copies using GP registers. */ +#define A_l r2 /* Call-clobbered. */ +#define A_h r3 /* Call-clobbered. */ +#define B_l r4 +#define B_h r5 +#define C_l r6 +#define C_h r7 +#define D_l r8 +#define D_h r9 +#endif + +/* Number of lines ahead to pre-fetch data. If you change this the code + below will need adjustment to compensate. */ + +#define prefetch_lines 5 + +#ifdef USE_VFP + .macro cpy_line_vfp vreg, base + vstr \vreg, [dst, #\base] + vldr \vreg, [src, #\base] + vstr d0, [dst, #\base + 8] + vldr d0, [src, #\base + 8] + vstr d1, [dst, #\base + 16] + vldr d1, [src, #\base + 16] + vstr d2, [dst, #\base + 24] + vldr d2, [src, #\base + 24] + vstr \vreg, [dst, #\base + 32] + vldr \vreg, [src, #\base + prefetch_lines * 64 - 32] + vstr d0, [dst, #\base + 40] + vldr d0, [src, #\base + 40] + vstr d1, [dst, #\base + 48] + vldr d1, [src, #\base + 48] + vstr d2, [dst, #\base + 56] + vldr d2, [src, #\base + 56] + .endm + + .macro cpy_tail_vfp vreg, base + vstr \vreg, [dst, #\base] + vldr \vreg, [src, #\base] + vstr d0, [dst, #\base + 8] + vldr d0, [src, #\base + 8] + vstr d1, [dst, #\base + 16] + vldr d1, [src, #\base + 16] + vstr d2, [dst, #\base + 24] + vldr d2, [src, #\base + 24] + vstr \vreg, [dst, #\base + 32] + vstr d0, [dst, #\base + 40] + vldr d0, [src, #\base + 40] + vstr d1, [dst, #\base + 48] + vldr d1, [src, #\base + 48] + vstr d2, [dst, #\base + 56] + vldr d2, [src, #\base + 56] + .endm +#endif ENTRY(__memcpy_chk) - cmp r2, r3 - bhi __memcpy_chk_fail + cmp r2, r3 + bhi __memcpy_chk_fail // Fall through to memcpy... END(__memcpy_chk) ENTRY(memcpy) - pld [r1, #64] - push {r0, lr} - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 + .p2align 6 + mov dst, dstin /* Preserve dstin, we need to return it. */ + cmp count, #64 + bge cpy_not_short + /* Deal with small copies quickly by dropping straight into the + exit block. */ + +tail63unaligned: +#ifdef USE_NEON + and tmp1, count, #0x38 + rsb tmp1, tmp1, #(56 - PC_OFFSET + INSN_SIZE) + add pc, pc, tmp1 + vld1.8 {d0}, [src]! /* 14 words to go. */ + vst1.8 {d0}, [dst]! + vld1.8 {d0}, [src]! /* 12 words to go. */ + vst1.8 {d0}, [dst]! + vld1.8 {d0}, [src]! /* 10 words to go. */ + vst1.8 {d0}, [dst]! + vld1.8 {d0}, [src]! /* 8 words to go. */ + vst1.8 {d0}, [dst]! + vld1.8 {d0}, [src]! /* 6 words to go. */ + vst1.8 {d0}, [dst]! + vld1.8 {d0}, [src]! /* 4 words to go. */ + vst1.8 {d0}, [dst]! + vld1.8 {d0}, [src]! /* 2 words to go. */ + vst1.8 {d0}, [dst]! + + tst count, #4 + ldrne tmp1, [src], #4 + strne tmp1, [dst], #4 +#else + /* Copy up to 15 full words of data. May not be aligned. */ + /* Cannot use VFP for unaligned data. */ + and tmp1, count, #0x3c + add dst, dst, tmp1 + add src, src, tmp1 + rsb tmp1, tmp1, #(60 - PC_OFFSET/2 + INSN_SIZE/2) + /* Jump directly into the sequence below at the correct offset. */ + add pc, pc, tmp1, lsl #1 + + ldr tmp1, [src, #-60] /* 15 words to go. */ + str tmp1, [dst, #-60] + + ldr tmp1, [src, #-56] /* 14 words to go. */ + str tmp1, [dst, #-56] + ldr tmp1, [src, #-52] + str tmp1, [dst, #-52] + + ldr tmp1, [src, #-48] /* 12 words to go. */ + str tmp1, [dst, #-48] + ldr tmp1, [src, #-44] + str tmp1, [dst, #-44] + + ldr tmp1, [src, #-40] /* 10 words to go. */ + str tmp1, [dst, #-40] + ldr tmp1, [src, #-36] + str tmp1, [dst, #-36] + + ldr tmp1, [src, #-32] /* 8 words to go. */ + str tmp1, [dst, #-32] + ldr tmp1, [src, #-28] + str tmp1, [dst, #-28] + + ldr tmp1, [src, #-24] /* 6 words to go. */ + str tmp1, [dst, #-24] + ldr tmp1, [src, #-20] + str tmp1, [dst, #-20] + + ldr tmp1, [src, #-16] /* 4 words to go. */ + str tmp1, [dst, #-16] + ldr tmp1, [src, #-12] + str tmp1, [dst, #-12] + + ldr tmp1, [src, #-8] /* 2 words to go. */ + str tmp1, [dst, #-8] + ldr tmp1, [src, #-4] + str tmp1, [dst, #-4] +#endif + + lsls count, count, #31 + ldrhcs tmp1, [src], #2 + ldrbne src, [src] /* Src is dead, use as a scratch. */ + strhcs tmp1, [dst], #2 + strbne src, [dst] + bx lr + +cpy_not_short: + /* At least 64 bytes to copy, but don't know the alignment yet. */ + str tmp2, [sp, #-FRAME_SIZE]! + and tmp2, src, #7 + and tmp1, dst, #7 + cmp tmp1, tmp2 + bne cpy_notaligned + +#ifdef USE_VFP + /* Magic dust alert! Force VFP on Cortex-A9. Experiments show + that the FP pipeline is much better at streaming loads and + stores. This is outside the critical loop. */ + vmov.f32 s0, s0 +#endif + + /* SRC and DST have the same mutual 64-bit alignment, but we may + still need to pre-copy some bytes to get to natural alignment. + We bring SRC and DST into full 64-bit alignment. */ + lsls tmp2, dst, #29 + beq 1f + rsbs tmp2, tmp2, #0 + sub count, count, tmp2, lsr #29 + ldrmi tmp1, [src], #4 + strmi tmp1, [dst], #4 + lsls tmp2, tmp2, #2 + ldrhcs tmp1, [src], #2 + ldrbne tmp2, [src], #1 + strhcs tmp1, [dst], #2 + strbne tmp2, [dst], #1 + +1: + subs tmp2, count, #64 /* Use tmp2 for count. */ + blt tail63aligned + + cmp tmp2, #512 + bge cpy_body_long + +cpy_body_medium: /* Count in tmp2. */ +#ifdef USE_VFP +1: + vldr d0, [src, #0] + subs tmp2, tmp2, #64 + vldr d1, [src, #8] + vstr d0, [dst, #0] + vldr d0, [src, #16] + vstr d1, [dst, #8] + vldr d1, [src, #24] + vstr d0, [dst, #16] + vldr d0, [src, #32] + vstr d1, [dst, #24] + vldr d1, [src, #40] + vstr d0, [dst, #32] + vldr d0, [src, #48] + vstr d1, [dst, #40] + vldr d1, [src, #56] + vstr d0, [dst, #48] + add src, src, #64 + vstr d1, [dst, #56] + add dst, dst, #64 + bge 1b + tst tmp2, #0x3f + beq done + +tail63aligned: /* Count in tmp2. */ + and tmp1, tmp2, #0x38 + add dst, dst, tmp1 + add src, src, tmp1 + rsb tmp1, tmp1, #(56 - PC_OFFSET + INSN_SIZE) + add pc, pc, tmp1 + + vldr d0, [src, #-56] /* 14 words to go. */ + vstr d0, [dst, #-56] + vldr d0, [src, #-48] /* 12 words to go. */ + vstr d0, [dst, #-48] + vldr d0, [src, #-40] /* 10 words to go. */ + vstr d0, [dst, #-40] + vldr d0, [src, #-32] /* 8 words to go. */ + vstr d0, [dst, #-32] + vldr d0, [src, #-24] /* 6 words to go. */ + vstr d0, [dst, #-24] + vldr d0, [src, #-16] /* 4 words to go. */ + vstr d0, [dst, #-16] + vldr d0, [src, #-8] /* 2 words to go. */ + vstr d0, [dst, #-8] +#else + sub src, src, #8 + sub dst, dst, #8 +1: + ldrd A_l, A_h, [src, #8] + strd A_l, A_h, [dst, #8] + ldrd A_l, A_h, [src, #16] + strd A_l, A_h, [dst, #16] + ldrd A_l, A_h, [src, #24] + strd A_l, A_h, [dst, #24] + ldrd A_l, A_h, [src, #32] + strd A_l, A_h, [dst, #32] + ldrd A_l, A_h, [src, #40] + strd A_l, A_h, [dst, #40] + ldrd A_l, A_h, [src, #48] + strd A_l, A_h, [dst, #48] + ldrd A_l, A_h, [src, #56] + strd A_l, A_h, [dst, #56] + ldrd A_l, A_h, [src, #64]! + strd A_l, A_h, [dst, #64]! + subs tmp2, tmp2, #64 + bge 1b + tst tmp2, #0x3f + bne 1f + ldr tmp2,[sp], #FRAME_SIZE + bx lr +1: + add src, src, #8 + add dst, dst, #8 + +tail63aligned: /* Count in tmp2. */ + /* Copy up to 7 d-words of data. Similar to Ltail63unaligned, but + we know that the src and dest are 64-bit aligned so we can use + LDRD/STRD to improve efficiency. */ + /* TMP2 is now negative, but we don't care about that. The bottom + six bits still tell us how many bytes are left to copy. */ + + and tmp1, tmp2, #0x38 + add dst, dst, tmp1 + add src, src, tmp1 + rsb tmp1, tmp1, #(56 - PC_OFFSET + INSN_SIZE) + add pc, pc, tmp1 + ldrd A_l, A_h, [src, #-56] /* 14 words to go. */ + strd A_l, A_h, [dst, #-56] + ldrd A_l, A_h, [src, #-48] /* 12 words to go. */ + strd A_l, A_h, [dst, #-48] + ldrd A_l, A_h, [src, #-40] /* 10 words to go. */ + strd A_l, A_h, [dst, #-40] + ldrd A_l, A_h, [src, #-32] /* 8 words to go. */ + strd A_l, A_h, [dst, #-32] + ldrd A_l, A_h, [src, #-24] /* 6 words to go. */ + strd A_l, A_h, [dst, #-24] + ldrd A_l, A_h, [src, #-16] /* 4 words to go. */ + strd A_l, A_h, [dst, #-16] + ldrd A_l, A_h, [src, #-8] /* 2 words to go. */ + strd A_l, A_h, [dst, #-8] + +#endif + tst tmp2, #4 + ldrne tmp1, [src], #4 + strne tmp1, [dst], #4 + lsls tmp2, tmp2, #31 /* Count (tmp2) now dead. */ + ldrhcs tmp1, [src], #2 + ldrbne tmp2, [src] + strhcs tmp1, [dst], #2 + strbne tmp2, [dst] + +done: + ldr tmp2, [sp], #FRAME_SIZE + bx lr + +cpy_body_long: /* Count in tmp2. */ + + /* Long copy. We know that there's at least (prefetch_lines * 64) + bytes to go. */ +#ifdef USE_VFP + /* Don't use PLD. Instead, read some data in advance of the current + copy position into a register. This should act like a PLD + operation but we won't have to repeat the transfer. */ + + vldr d3, [src, #0] + vldr d4, [src, #64] + vldr d5, [src, #128] + vldr d6, [src, #192] + vldr d7, [src, #256] + + vldr d0, [src, #8] + vldr d1, [src, #16] + vldr d2, [src, #24] + add src, src, #32 + + subs tmp2, tmp2, #prefetch_lines * 64 * 2 + blt 2f +1: + cpy_line_vfp d3, 0 + cpy_line_vfp d4, 64 + cpy_line_vfp d5, 128 + add dst, dst, #3 * 64 + add src, src, #3 * 64 + cpy_line_vfp d6, 0 + cpy_line_vfp d7, 64 + add dst, dst, #2 * 64 + add src, src, #2 * 64 + subs tmp2, tmp2, #prefetch_lines * 64 + bge 1b + +2: + cpy_tail_vfp d3, 0 + cpy_tail_vfp d4, 64 + cpy_tail_vfp d5, 128 + add src, src, #3 * 64 + add dst, dst, #3 * 64 + cpy_tail_vfp d6, 0 + vstr d7, [dst, #64] + vldr d7, [src, #64] + vstr d0, [dst, #64 + 8] + vldr d0, [src, #64 + 8] + vstr d1, [dst, #64 + 16] + vldr d1, [src, #64 + 16] + vstr d2, [dst, #64 + 24] + vldr d2, [src, #64 + 24] + vstr d7, [dst, #64 + 32] + add src, src, #96 + vstr d0, [dst, #64 + 40] + vstr d1, [dst, #64 + 48] + vstr d2, [dst, #64 + 56] + add dst, dst, #128 + add tmp2, tmp2, #prefetch_lines * 64 + b cpy_body_medium +#else + /* Long copy. Use an SMS style loop to maximize the I/O + bandwidth of the core. We don't have enough spare registers + to synthesise prefetching, so use PLD operations. */ + /* Pre-bias src and dst. */ + sub src, src, #8 + sub dst, dst, #8 + pld [src, #8] + pld [src, #72] + subs tmp2, tmp2, #64 + pld [src, #136] + ldrd A_l, A_h, [src, #8] + strd B_l, B_h, [sp, #8] + ldrd B_l, B_h, [src, #16] + strd C_l, C_h, [sp, #16] + ldrd C_l, C_h, [src, #24] + strd D_l, D_h, [sp, #24] + pld [src, #200] + ldrd D_l, D_h, [src, #32]! + b 1f + .p2align 6 +2: + pld [src, #232] + strd A_l, A_h, [dst, #40] + ldrd A_l, A_h, [src, #40] + strd B_l, B_h, [dst, #48] + ldrd B_l, B_h, [src, #48] + strd C_l, C_h, [dst, #56] + ldrd C_l, C_h, [src, #56] + strd D_l, D_h, [dst, #64]! + ldrd D_l, D_h, [src, #64]! + subs tmp2, tmp2, #64 +1: + strd A_l, A_h, [dst, #8] + ldrd A_l, A_h, [src, #8] + strd B_l, B_h, [dst, #16] + ldrd B_l, B_h, [src, #16] + strd C_l, C_h, [dst, #24] + ldrd C_l, C_h, [src, #24] + strd D_l, D_h, [dst, #32] + ldrd D_l, D_h, [src, #32] + bcs 2b + /* Save the remaining bytes and restore the callee-saved regs. */ + strd A_l, A_h, [dst, #40] + add src, src, #40 + strd B_l, B_h, [dst, #48] + ldrd B_l, B_h, [sp, #8] + strd C_l, C_h, [dst, #56] + ldrd C_l, C_h, [sp, #16] + strd D_l, D_h, [dst, #64] + ldrd D_l, D_h, [sp, #24] + add dst, dst, #72 + tst tmp2, #0x3f + bne tail63aligned + ldr tmp2, [sp], #FRAME_SIZE + bx lr +#endif + +cpy_notaligned: + pld [src] + pld [src, #64] + /* There's at least 64 bytes to copy, but there is no mutual + alignment. */ + /* Bring DST to 64-bit alignment. */ + lsls tmp2, dst, #29 + pld [src, #(2 * 64)] + beq 1f + rsbs tmp2, tmp2, #0 + sub count, count, tmp2, lsr #29 + ldrmi tmp1, [src], #4 + strmi tmp1, [dst], #4 + lsls tmp2, tmp2, #2 + ldrbne tmp1, [src], #1 + ldrhcs tmp2, [src], #2 + strbne tmp1, [dst], #1 + strhcs tmp2, [dst], #2 +1: + pld [src, #(3 * 64)] + subs count, count, #64 + ldrmi tmp2, [sp], #FRAME_SIZE + bmi tail63unaligned + pld [src, #(4 * 64)] + +#ifdef USE_NEON + vld1.8 {d0-d3}, [src]! + vld1.8 {d4-d7}, [src]! + subs count, count, #64 + bmi 2f +1: + pld [src, #(4 * 64)] + vst1.8 {d0-d3}, [ALIGN (dst, 64)]! + vld1.8 {d0-d3}, [src]! + vst1.8 {d4-d7}, [ALIGN (dst, 64)]! + vld1.8 {d4-d7}, [src]! + subs count, count, #64 + bpl 1b +2: + vst1.8 {d0-d3}, [ALIGN (dst, 64)]! + vst1.8 {d4-d7}, [ALIGN (dst, 64)]! + ands count, count, #0x3f +#else + /* Use an SMS style loop to maximize the I/O bandwidth. */ + sub src, src, #4 + sub dst, dst, #8 + subs tmp2, count, #64 /* Use tmp2 for count. */ + ldr A_l, [src, #4] + ldr A_h, [src, #8] + strd B_l, B_h, [sp, #8] + ldr B_l, [src, #12] + ldr B_h, [src, #16] + strd C_l, C_h, [sp, #16] + ldr C_l, [src, #20] + ldr C_h, [src, #24] + strd D_l, D_h, [sp, #24] + ldr D_l, [src, #28] + ldr D_h, [src, #32]! + b 1f + .p2align 6 +2: + pld [src, #(5 * 64) - (32 - 4)] + strd A_l, A_h, [dst, #40] + ldr A_l, [src, #36] + ldr A_h, [src, #40] + strd B_l, B_h, [dst, #48] + ldr B_l, [src, #44] + ldr B_h, [src, #48] + strd C_l, C_h, [dst, #56] + ldr C_l, [src, #52] + ldr C_h, [src, #56] + strd D_l, D_h, [dst, #64]! + ldr D_l, [src, #60] + ldr D_h, [src, #64]! + subs tmp2, tmp2, #64 +1: + strd A_l, A_h, [dst, #8] + ldr A_l, [src, #4] + ldr A_h, [src, #8] + strd B_l, B_h, [dst, #16] + ldr B_l, [src, #12] + ldr B_h, [src, #16] + strd C_l, C_h, [dst, #24] + ldr C_l, [src, #20] + ldr C_h, [src, #24] + strd D_l, D_h, [dst, #32] + ldr D_l, [src, #28] + ldr D_h, [src, #32] + bcs 2b + + /* Save the remaining bytes and restore the callee-saved regs. */ + strd A_l, A_h, [dst, #40] + add src, src, #36 + strd B_l, B_h, [dst, #48] + ldrd B_l, B_h, [sp, #8] + strd C_l, C_h, [dst, #56] + ldrd C_l, C_h, [sp, #16] + strd D_l, D_h, [dst, #64] + ldrd D_l, D_h, [sp, #24] + add dst, dst, #72 + ands count, tmp2, #0x3f +#endif + ldr tmp2, [sp], #FRAME_SIZE + bne tail63unaligned + bx lr END(memcpy) #define MEMCPY_BASE __memcpy_base From 60a7b89100ff12ef19d7e92397cb99d43277753e Mon Sep 17 00:00:00 2001 From: Jake Weinstein Date: Sat, 15 Nov 2014 12:01:42 -0500 Subject: [PATCH 30/89] krait: use A15 memchr Change-Id: I6d9c773612db02706d5a36252eb7d4458d9b6e1b --- libc/arch-arm/krait/krait.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/arch-arm/krait/krait.mk b/libc/arch-arm/krait/krait.mk index 5947dcb7b..47389a9b3 100644 --- a/libc/arch-arm/krait/krait.mk +++ b/libc/arch-arm/krait/krait.mk @@ -1,5 +1,5 @@ libc_bionic_src_files_arm += \ - bionic/memchr.c \ + arch-arm/cortex-a15/bionic/memchr.S \ arch-arm/krait/bionic/memcpy.S \ arch-arm/krait/bionic/memset.S \ arch-arm/krait/bionic/strcmp.S \ From 99f72eacd9fafa251bddfc5996f38f6b7677b754 Mon Sep 17 00:00:00 2001 From: Ziyan Date: Tue, 25 Nov 2014 22:57:49 +0100 Subject: [PATCH 31/89] Revert "Remove the MD5 implementation that was only used by toolbox." This reverts commit 67401638a92abd52e5896183e851b1b097da07ac. Added TARGET_NEEDS_BIONIC_MD5 to indicate if a device needs it. In particular, the Galaxy Nexus RIL breaks without this. Change-Id: I49f1779566dfeb5093970ed0bbda741919318a83 --- libc/Android.mk | 4 + libc/bionic/md5.c | 279 ++++++++++++++++++++++++++++++++++++++++++++++ libc/bionic/md5.h | 50 +++++++++ 3 files changed, 333 insertions(+) create mode 100644 libc/bionic/md5.c create mode 100644 libc/bionic/md5.h diff --git a/libc/Android.mk b/libc/Android.mk index 8e1e2e462..fb99c81d1 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -65,6 +65,10 @@ libc_common_src_files := \ stdio/snprintf.c\ stdio/sprintf.c \ +ifeq ($(TARGET_NEEDS_BIONIC_MD5),true) +libc_common_src_files += bionic/md5.c +endif + # Fortify implementations of libc functions. libc_common_src_files += \ bionic/__FD_chk.cpp \ diff --git a/libc/bionic/md5.c b/libc/bionic/md5.c new file mode 100644 index 000000000..ba4aaed22 --- /dev/null +++ b/libc/bionic/md5.c @@ -0,0 +1,279 @@ +/* + * Copyright (c) 1995 - 2001 Kungliga Tekniska Högskolan + * (Royal Institute of Technology, Stockholm, Sweden). + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the Institute nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" + +__RCSID("$Heimdal: md5.c,v 1.15 2001/01/29 04:33:44 assar Exp $" + "$NetBSD: md5.c,v 1.1.1.4 2002/09/12 12:41:42 joda Exp $"); +#endif + +#include +#include "md5.h" +#include "hash.h" + +#define A m->counter[0] +#define B m->counter[1] +#define C m->counter[2] +#define D m->counter[3] +#define X data + +void +MD5_Init (struct md5 *m) +{ + m->sz[0] = 0; + m->sz[1] = 0; + D = 0x10325476; + C = 0x98badcfe; + B = 0xefcdab89; + A = 0x67452301; +} + +#define F(x,y,z) CRAYFIX((x & y) | (~x & z)) +#define G(x,y,z) CRAYFIX((x & z) | (y & ~z)) +#define H(x,y,z) (x ^ y ^ z) +#define I(x,y,z) CRAYFIX(y ^ (x | ~z)) + +#define DOIT(a,b,c,d,k,s,i,OP) \ +a = b + cshift(a + OP(b,c,d) + X[k] + (i), s) + +#define DO1(a,b,c,d,k,s,i) DOIT(a,b,c,d,k,s,i,F) +#define DO2(a,b,c,d,k,s,i) DOIT(a,b,c,d,k,s,i,G) +#define DO3(a,b,c,d,k,s,i) DOIT(a,b,c,d,k,s,i,H) +#define DO4(a,b,c,d,k,s,i) DOIT(a,b,c,d,k,s,i,I) + +static inline void +calc (struct md5 *m, u_int32_t *data) +{ + u_int32_t AA, BB, CC, DD; + + AA = A; + BB = B; + CC = C; + DD = D; + + /* Round 1 */ + + DO1(A,B,C,D,0,7,0xd76aa478); + DO1(D,A,B,C,1,12,0xe8c7b756); + DO1(C,D,A,B,2,17,0x242070db); + DO1(B,C,D,A,3,22,0xc1bdceee); + + DO1(A,B,C,D,4,7,0xf57c0faf); + DO1(D,A,B,C,5,12,0x4787c62a); + DO1(C,D,A,B,6,17,0xa8304613); + DO1(B,C,D,A,7,22,0xfd469501); + + DO1(A,B,C,D,8,7,0x698098d8); + DO1(D,A,B,C,9,12,0x8b44f7af); + DO1(C,D,A,B,10,17,0xffff5bb1); + DO1(B,C,D,A,11,22,0x895cd7be); + + DO1(A,B,C,D,12,7,0x6b901122); + DO1(D,A,B,C,13,12,0xfd987193); + DO1(C,D,A,B,14,17,0xa679438e); + DO1(B,C,D,A,15,22,0x49b40821); + + /* Round 2 */ + + DO2(A,B,C,D,1,5,0xf61e2562); + DO2(D,A,B,C,6,9,0xc040b340); + DO2(C,D,A,B,11,14,0x265e5a51); + DO2(B,C,D,A,0,20,0xe9b6c7aa); + + DO2(A,B,C,D,5,5,0xd62f105d); + DO2(D,A,B,C,10,9,0x2441453); + DO2(C,D,A,B,15,14,0xd8a1e681); + DO2(B,C,D,A,4,20,0xe7d3fbc8); + + DO2(A,B,C,D,9,5,0x21e1cde6); + DO2(D,A,B,C,14,9,0xc33707d6); + DO2(C,D,A,B,3,14,0xf4d50d87); + DO2(B,C,D,A,8,20,0x455a14ed); + + DO2(A,B,C,D,13,5,0xa9e3e905); + DO2(D,A,B,C,2,9,0xfcefa3f8); + DO2(C,D,A,B,7,14,0x676f02d9); + DO2(B,C,D,A,12,20,0x8d2a4c8a); + + /* Round 3 */ + + DO3(A,B,C,D,5,4,0xfffa3942); + DO3(D,A,B,C,8,11,0x8771f681); + DO3(C,D,A,B,11,16,0x6d9d6122); + DO3(B,C,D,A,14,23,0xfde5380c); + + DO3(A,B,C,D,1,4,0xa4beea44); + DO3(D,A,B,C,4,11,0x4bdecfa9); + DO3(C,D,A,B,7,16,0xf6bb4b60); + DO3(B,C,D,A,10,23,0xbebfbc70); + + DO3(A,B,C,D,13,4,0x289b7ec6); + DO3(D,A,B,C,0,11,0xeaa127fa); + DO3(C,D,A,B,3,16,0xd4ef3085); + DO3(B,C,D,A,6,23,0x4881d05); + + DO3(A,B,C,D,9,4,0xd9d4d039); + DO3(D,A,B,C,12,11,0xe6db99e5); + DO3(C,D,A,B,15,16,0x1fa27cf8); + DO3(B,C,D,A,2,23,0xc4ac5665); + + /* Round 4 */ + + DO4(A,B,C,D,0,6,0xf4292244); + DO4(D,A,B,C,7,10,0x432aff97); + DO4(C,D,A,B,14,15,0xab9423a7); + DO4(B,C,D,A,5,21,0xfc93a039); + + DO4(A,B,C,D,12,6,0x655b59c3); + DO4(D,A,B,C,3,10,0x8f0ccc92); + DO4(C,D,A,B,10,15,0xffeff47d); + DO4(B,C,D,A,1,21,0x85845dd1); + + DO4(A,B,C,D,8,6,0x6fa87e4f); + DO4(D,A,B,C,15,10,0xfe2ce6e0); + DO4(C,D,A,B,6,15,0xa3014314); + DO4(B,C,D,A,13,21,0x4e0811a1); + + DO4(A,B,C,D,4,6,0xf7537e82); + DO4(D,A,B,C,11,10,0xbd3af235); + DO4(C,D,A,B,2,15,0x2ad7d2bb); + DO4(B,C,D,A,9,21,0xeb86d391); + + A += AA; + B += BB; + C += CC; + D += DD; +} + +/* + * From `Performance analysis of MD5' by Joseph D. Touch + */ +#if !defined(__BYTE_ORDER) || !defined (__BIG_ENDIAN) +#error __BYTE_ORDER macros not defined +#endif + +#if __BYTE_ORDER == __BIG_ENDIAN +static inline u_int32_t +swap_u_int32_t (u_int32_t t) +{ + u_int32_t temp1, temp2; + + temp1 = cshift(t, 16); + temp2 = temp1 >> 8; + temp1 &= 0x00ff00ff; + temp2 &= 0x00ff00ff; + temp1 <<= 8; + return temp1 | temp2; +} +#endif + +struct x32{ + unsigned int a:32; + unsigned int b:32; +}; + +void +MD5_Update (struct md5 *m, const void *v, size_t len) +{ + const unsigned char *p = v; + size_t old_sz = m->sz[0]; + size_t offset; + + m->sz[0] += len * 8; + if (m->sz[0] < old_sz) + ++m->sz[1]; + offset = (old_sz / 8) % 64; + while(len > 0){ + size_t l = min(len, 64 - offset); + memcpy(m->save + offset, p, l); + offset += l; + p += l; + len -= l; + if(offset == 64){ +#if __BYTE_ORDER == __BIG_ENDIAN + int i; + u_int32_t current[16]; + struct x32 *u = (struct x32*)m->save; + for(i = 0; i < 8; i++){ + current[2*i+0] = swap_u_int32_t(u[i].a); + current[2*i+1] = swap_u_int32_t(u[i].b); + } + calc(m, current); +#else + calc(m, (u_int32_t*)m->save); +#endif + offset = 0; + } + } +} + +void +MD5_Final (void *res, struct md5 *m) +{ + unsigned char zeros[72]; + unsigned offset = (m->sz[0] / 8) % 64; + unsigned int dstart = (120 - offset - 1) % 64 + 1; + + *zeros = 0x80; + memset (zeros + 1, 0, sizeof(zeros) - 1); + zeros[dstart+0] = (m->sz[0] >> 0) & 0xff; + zeros[dstart+1] = (m->sz[0] >> 8) & 0xff; + zeros[dstart+2] = (m->sz[0] >> 16) & 0xff; + zeros[dstart+3] = (m->sz[0] >> 24) & 0xff; + zeros[dstart+4] = (m->sz[1] >> 0) & 0xff; + zeros[dstart+5] = (m->sz[1] >> 8) & 0xff; + zeros[dstart+6] = (m->sz[1] >> 16) & 0xff; + zeros[dstart+7] = (m->sz[1] >> 24) & 0xff; + MD5_Update (m, zeros, dstart + 8); + { + int i; + unsigned char *r = (unsigned char *)res; + + for (i = 0; i < 4; ++i) { + r[4*i] = m->counter[i] & 0xFF; + r[4*i+1] = (m->counter[i] >> 8) & 0xFF; + r[4*i+2] = (m->counter[i] >> 16) & 0xFF; + r[4*i+3] = (m->counter[i] >> 24) & 0xFF; + } + } +#if 0 + { + int i; + u_int32_t *r = (u_int32_t *)res; + + for (i = 0; i < 4; ++i) + r[i] = swap_u_int32_t (m->counter[i]); + } +#endif +} diff --git a/libc/bionic/md5.h b/libc/bionic/md5.h new file mode 100644 index 000000000..a381994ef --- /dev/null +++ b/libc/bionic/md5.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 1995 - 2001 Kungliga Tekniska Högskolan + * (Royal Institute of Technology, Stockholm, Sweden). + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the name of the Institute nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* $Heimdal: md5.h,v 1.8 2001/01/29 02:08:57 assar Exp $ + $NetBSD: md5.h,v 1.1.1.4 2002/09/12 12:41:42 joda Exp $ */ + +#include +#include + +struct md5 { + unsigned int sz[2]; + u_int32_t counter[4]; + unsigned char save[64]; +}; + +typedef struct md5 MD5_CTX; + +void MD5_Init (struct md5 *m); +void MD5_Update (struct md5 *m, const void *p, size_t len); +void MD5_Final (void *res, struct md5 *m); /* u_int32_t res[4] */ From 9e695b86219d9ae960f35d900aa78573bb8df2f7 Mon Sep 17 00:00:00 2001 From: Steve Kondik Date: Fri, 9 Aug 2013 21:56:10 -0700 Subject: [PATCH 32/89] libc: Add Scorpion-optimized memcpy and memset * Optimized routines from Qualcomm for msm8660 class SoCs * Enable with TARGET_CPU_VARIANT := scorpion * Use cortex-a15 memcpy and krait memset routines Change-Id: I01d0f22efba5a418ddd20fca0d0c570d855e0f6f --- libc/arch-arm/arm.mk | 4 +++- libc/arch-arm/scorpion/scorpion.mk | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 libc/arch-arm/scorpion/scorpion.mk diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index d4ca00783..3bfa7a946 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -72,14 +72,16 @@ ifeq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),) $(warning TARGET_$(my_2nd_arch_prefix)ARCH is arm, but TARGET_$(my_2nd_arch_prefix)CPU_VARIANT is not defined) endif ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),krait) +ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),scorpion) ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),cortex-a53) libc_bionic_src_files_arm += \ upstream-openbsd/lib/libc/string/bcopy.c endif endif +endif cpu_variant_mk := $(LOCAL_PATH)/arch-arm/$(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)/$(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT).mk ifeq ($(wildcard $(cpu_variant_mk)),) -$(error "TARGET_$(my_2nd_arch_prefix)CPU_VARIANT not set or set to an unknown value. Possible values are cortex-a7, cortex-a8, cortex-a9, cortex-a15, krait, denver. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.") +$(error "TARGET_$(my_2nd_arch_prefix)CPU_VARIANT not set or set to an unknown value. Possible values are cortex-a7, cortex-a8, cortex-a9, cortex-a15, krait, scorpion, denver. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.") endif include $(cpu_variant_mk) libc_common_additional_dependencies += $(cpu_variant_mk) diff --git a/libc/arch-arm/scorpion/scorpion.mk b/libc/arch-arm/scorpion/scorpion.mk new file mode 100644 index 000000000..8bd3a76f9 --- /dev/null +++ b/libc/arch-arm/scorpion/scorpion.mk @@ -0,0 +1,15 @@ +# Use krait versions of memset/strcmp/memmove +libc_bionic_src_files_arm += \ + arch-arm/krait/bionic/memset.S \ + arch-arm/krait/bionic/strcmp.S \ + arch-arm/krait/bionic/memmove.S + +# Use cortex-a15 versions of memcpy/strcat/strcpy/strlen +libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memchr.S \ + arch-arm/cortex-a15/bionic/memcpy.S \ + arch-arm/cortex-a15/bionic/strcat.S \ + arch-arm/cortex-a15/bionic/strcpy.S \ + arch-arm/cortex-a15/bionic/strlen.S \ + arch-arm/cortex-a15/bionic/__strcat_chk.S \ + arch-arm/cortex-a15/bionic/__strcpy_chk.S \ From 07bf4fc71588d249d732585d3425989696d636a7 Mon Sep 17 00:00:00 2001 From: "Howard M. Harte" Date: Fri, 28 Mar 2014 11:32:50 +0530 Subject: [PATCH 33/89] libc: Add memchr for cortex-a53 Change-Id: Id84695006991bb9aaef8ddc127c3bee646948527 --- libc/arch-arm/cortex-a53/cortex-a53.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/libc/arch-arm/cortex-a53/cortex-a53.mk b/libc/arch-arm/cortex-a53/cortex-a53.mk index 3ed80f283..c9ab31f42 100644 --- a/libc/arch-arm/cortex-a53/cortex-a53.mk +++ b/libc/arch-arm/cortex-a53/cortex-a53.mk @@ -1,4 +1,5 @@ libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memchr.S \ arch-arm/cortex-a15/bionic/memcpy.S \ arch-arm/cortex-a15/bionic/memset.S \ arch-arm/cortex-a15/bionic/strcat.S \ From 40115a3e505b34a7fdb0ad1480139cf10ddda0c1 Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Wed, 3 Dec 2014 14:39:20 -0800 Subject: [PATCH 34/89] Implement . Based on the package/apps/Terminal implementation. I'll switch them over shortly. This also lets us build the toybox version of netcat. Change-Id: Ia922a100141a67409264b43b937eeca07b21f344 --- libc/bionic/pty.cpp | 86 +++++++++++++++++++++++++++++++++++++++++++-- libc/include/pty.h | 44 +++++++++++++++++++++++ libc/include/utmp.h | 2 ++ tests/Android.mk | 4 ++- tests/pty_test.cpp | 67 +++++++++++++++++++++++++++++++++++ tests/utmp_test.cpp | 25 +++++++++++++ 6 files changed, 225 insertions(+), 3 deletions(-) create mode 100644 libc/include/pty.h create mode 100644 tests/pty_test.cpp create mode 100644 tests/utmp_test.cpp diff --git a/libc/bionic/pty.cpp b/libc/bionic/pty.cpp index 995e00608..2c8618047 100644 --- a/libc/bionic/pty.cpp +++ b/libc/bionic/pty.cpp @@ -28,13 +28,15 @@ #include #include +#include #include #include #include #include #include +#include -int getpt(void) { +int getpt() { return posix_openpt(O_RDWR|O_NOCTTY); } @@ -47,7 +49,7 @@ int posix_openpt(int flags) { } char* ptsname(int fd) { - static char buf[64]; + static char buf[32]; return ptsname_r(fd, buf, sizeof(buf)) == 0 ? buf : NULL; } @@ -105,3 +107,83 @@ int unlockpt(int fd) { int unlock = 0; return ioctl(fd, TIOCSPTLCK, &unlock); } + +int openpty(int* master, int* slave, char* name, const termios* t, const winsize* ws) { + *master = getpt(); + if (*master == -1) { + return -1; + } + + if (grantpt(*master) == -1 || unlockpt(*master) == -1) { + close(*master); + return -1; + } + + char buf[32]; + if (name == NULL) { + name = buf; + } + if (ptsname_r(*master, name, sizeof(buf)) != 0) { + close(*master); + return -1; + } + + *slave = open(name, O_RDWR|O_NOCTTY); + if (*slave == -1) { + close(*master); + return -1; + } + + if (t != NULL) { + tcsetattr(*slave, TCSAFLUSH, t); + } + if (ws != NULL) { + ioctl(*slave, TIOCSWINSZ, ws); + } + + return 0; +} + +int forkpty(int* master, char* name, const termios* t, const winsize* ws) { + int slave; + if (openpty(master, &slave, name, t, ws) == -1) { + return -1; + } + + pid_t pid = fork(); + if (pid == -1) { + close(*master); + close(slave); + return -1; + } + + if (pid == 0) { + // Child. + close(*master); + if (login_tty(slave) == -1) { + _exit(1); + } + return 0; + } + + // Parent. + close(slave); + return pid; +} + +int login_tty(int fd) { + setsid(); + + if (ioctl(fd, TIOCSCTTY, NULL) == -1) { + return -1; + } + + dup2(fd, STDIN_FILENO); + dup2(fd, STDOUT_FILENO); + dup2(fd, STDERR_FILENO); + if (fd > STDERR_FILENO) { + close(fd); + } + + return 0; +} diff --git a/libc/include/pty.h b/libc/include/pty.h new file mode 100644 index 000000000..bca113752 --- /dev/null +++ b/libc/include/pty.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _PTY_H +#define _PTY_H + +#include + +#include +#include + +__BEGIN_DECLS + +int openpty(int*, int*, char*, const struct termios*, const struct winsize*); +int forkpty(int*, char*, const struct termios*, const struct winsize*); + +__END_DECLS + +#endif /* _PTY_H */ diff --git a/libc/include/utmp.h b/libc/include/utmp.h index d76422781..ebf23726a 100644 --- a/libc/include/utmp.h +++ b/libc/include/utmp.h @@ -91,6 +91,8 @@ int utmpname(const char*); void setutent(); struct utmp* getutent(); +int login_tty(int); + __END_DECLS #endif /* _UTMP_H_ */ diff --git a/tests/Android.mk b/tests/Android.mk index 8b0b0a0e0..6eeeacbf4 100644 --- a/tests/Android.mk +++ b/tests/Android.mk @@ -85,6 +85,7 @@ libBionicStandardTests_src_files := \ mntent_test.cpp \ netdb_test.cpp \ pthread_test.cpp \ + pty_test.cpp \ regex_test.cpp \ sched_test.cpp \ search_test.cpp \ @@ -116,6 +117,7 @@ libBionicStandardTests_src_files := \ uchar_test.cpp \ uniqueptr_test.cpp \ unistd_test.cpp \ + utmp_test.cpp \ wchar_test.cpp \ libBionicStandardTests_cflags := \ @@ -291,7 +293,7 @@ bionic-unit-tests-glibc_whole_static_libraries := \ libBionicStandardTests \ bionic-unit-tests-glibc_ldlibs := \ - -lrt -ldl \ + -lrt -ldl -lutil \ bionic-unit-tests-glibc_c_includes := \ bionic/libc \ diff --git a/tests/pty_test.cpp b/tests/pty_test.cpp new file mode 100644 index 000000000..7fe97e9af --- /dev/null +++ b/tests/pty_test.cpp @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +TEST(pty, openpty) { + int master, slave; + char name[32]; + struct winsize w = { 123, 456, 9999, 999 }; + ASSERT_EQ(0, openpty(&master, &slave, name, NULL, &w)); + ASSERT_NE(-1, master); + ASSERT_NE(-1, slave); + ASSERT_NE(master, slave); + + char tty_name[32]; + ASSERT_EQ(0, ttyname_r(slave, tty_name, sizeof(tty_name))); + ASSERT_STREQ(tty_name, name); + + struct winsize w_actual; + ASSERT_EQ(0, ioctl(slave, TIOCGWINSZ, &w_actual)); + ASSERT_EQ(w_actual.ws_row, w.ws_row); + ASSERT_EQ(w_actual.ws_col, w.ws_col); + ASSERT_EQ(w_actual.ws_xpixel, w.ws_xpixel); + ASSERT_EQ(w_actual.ws_ypixel, w.ws_ypixel); + + close(master); + close(slave); +} + +TEST(pty, forkpty) { + pid_t sid = getsid(0); + + int master; + pid_t pid = forkpty(&master, NULL, NULL, NULL); + ASSERT_NE(-1, pid); + + if (pid == 0) { + // We're the child. + ASSERT_NE(sid, getsid(0)); + _exit(0); + } + + ASSERT_EQ(sid, getsid(0)); + + int status; + ASSERT_EQ(pid, waitpid(pid, &status, 0)); + ASSERT_TRUE(WIFEXITED(status)); + ASSERT_EQ(0, WEXITSTATUS(status)); + + close(master); +} diff --git a/tests/utmp_test.cpp b/tests/utmp_test.cpp new file mode 100644 index 000000000..b61110d68 --- /dev/null +++ b/tests/utmp_test.cpp @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +TEST(utmp, login_tty) { + // login_tty is tested indirectly by the openpty and forkpty tests. + // This test just checks that we're exporting the symbol independently. + ASSERT_EQ(-1, login_tty(-1)); +} From 0f0be6d94fd660a5584645919415b431ffda1623 Mon Sep 17 00:00:00 2001 From: Jake Weinstein Date: Fri, 12 Dec 2014 00:39:19 +0100 Subject: [PATCH 35/89] libc: denver: use Cortex-A15 Memchr Change-Id: I6c2dc07ca6faf0c0a3cfe684581ae99b103e41b7 --- libc/arch-arm/denver/denver.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libc/arch-arm/denver/denver.mk b/libc/arch-arm/denver/denver.mk index 0bc52a27b..61c55a193 100644 --- a/libc/arch-arm/denver/denver.mk +++ b/libc/arch-arm/denver/denver.mk @@ -5,8 +5,9 @@ libc_bionic_src_files_arm += \ arch-arm/denver/bionic/__strcat_chk.S \ arch-arm/denver/bionic/__strcpy_chk.S \ -# Use cortex-a15 versions of strcat/strcpy/strlen. +# Use cortex-a15 versions of memchr/strcat/strcpy/strlen. libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memchr.S \ arch-arm/cortex-a15/bionic/stpcpy.S \ arch-arm/cortex-a15/bionic/strcat.S \ arch-arm/cortex-a15/bionic/strcmp.S \ From e2194aa88c64991c79d3b825d311b6b5ce93af43 Mon Sep 17 00:00:00 2001 From: Tom Marshall Date: Fri, 30 Jan 2015 15:14:52 -0800 Subject: [PATCH 36/89] bionic: Allow alternate factory.prop path Change-Id: Ia60d625c47d9aaceee42c30419a2f04ac1985eb9 --- libc/include/sys/_system_properties.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libc/include/sys/_system_properties.h b/libc/include/sys/_system_properties.h index 0349e4c3d..0a842bd7c 100644 --- a/libc/include/sys/_system_properties.h +++ b/libc/include/sys/_system_properties.h @@ -83,7 +83,11 @@ struct prop_msg #define PROP_PATH_SYSTEM_DEFAULT "/system/default.prop" #define PROP_PATH_VENDOR_BUILD "/vendor/build.prop" #define PROP_PATH_LOCAL_OVERRIDE "/data/local.prop" +#ifdef OVERRIDE_PROP_PATH_FACTORY +#define PROP_PATH_FACTORY OVERRIDE_PROP_PATH_FACTORY +#else #define PROP_PATH_FACTORY "/factory/factory.prop" +#endif /* ** Map the property area from the specified filename. This From f7852274b68184fc822213c22b8fee8fdb55e0ce Mon Sep 17 00:00:00 2001 From: Raghavendra Prasad Nagaraj Date: Mon, 19 Jan 2015 19:24:53 +0530 Subject: [PATCH 37/89] libc: memcpy_base: Disable opt for certain targets memcpy_base.S optimization is yielding low memory scores for 8x26 and 8x10. So this opt is being conditionally compiled. CRs-fixed: 790279 Change-Id: I5a5a60dfa81d86fd43b6be73fa344da861ebb33c --- libc/arch-arm/krait/krait.mk | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/libc/arch-arm/krait/krait.mk b/libc/arch-arm/krait/krait.mk index 47389a9b3..8bf210035 100644 --- a/libc/arch-arm/krait/krait.mk +++ b/libc/arch-arm/krait/krait.mk @@ -1,12 +1,19 @@ libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/memchr.S \ - arch-arm/krait/bionic/memcpy.S \ arch-arm/krait/bionic/memset.S \ arch-arm/krait/bionic/strcmp.S \ arch-arm/krait/bionic/__strcat_chk.S \ arch-arm/krait/bionic/__strcpy_chk.S \ arch-arm/krait/bionic/memmove.S +#For some targets we don't need this optimization +ifeq ($(TARGET_CPU_MEMCPY_BASE_OPT_DISABLE),true) +libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memcpy.S +else +libc_bionic_src_files_arm += \ + arch-arm/krait/bionic/memcpy.S +endif # Use cortex-a15 versions of strcat/strcpy/strlen and standard memmove libc_bionic_src_files_arm += \ From fcb430b4082bdd399e1e2bf9e5a2a2cb8c5c184d Mon Sep 17 00:00:00 2001 From: Michael Bestas Date: Sat, 21 Mar 2015 02:47:26 +0200 Subject: [PATCH 38/89] libc: scorpion: Use cortex-a15 stpcpy * Like krait, fixes compilation on 5.1 Change-Id: I08989c0206c333f7fd0f938b6a6d6b26475d63d8 --- libc/arch-arm/scorpion/scorpion.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/libc/arch-arm/scorpion/scorpion.mk b/libc/arch-arm/scorpion/scorpion.mk index 8bd3a76f9..74abffd21 100644 --- a/libc/arch-arm/scorpion/scorpion.mk +++ b/libc/arch-arm/scorpion/scorpion.mk @@ -8,6 +8,7 @@ libc_bionic_src_files_arm += \ libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/memchr.S \ arch-arm/cortex-a15/bionic/memcpy.S \ + arch-arm/cortex-a15/bionic/stpcpy.S \ arch-arm/cortex-a15/bionic/strcat.S \ arch-arm/cortex-a15/bionic/strcpy.S \ arch-arm/cortex-a15/bionic/strlen.S \ From 62d970169f8fcc114b79c537624c6f552c01d5a3 Mon Sep 17 00:00:00 2001 From: Sudhir Sharma Date: Tue, 10 Mar 2015 12:34:45 -0700 Subject: [PATCH 39/89] Compilation Fix to build. This change is to fix the compilation. Change-Id: Ic51e0c0687d566a323b4c4e02bc2c999d85ecd0f --- libc/arch-arm/arm.mk | 1 - libc/arch-arm/cortex-a53/cortex-a53.mk | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index 3bfa7a946..ab82ec5f9 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -37,7 +37,6 @@ libc_freebsd_src_files_arm += \ upstream-freebsd/lib/libc/string/wcsrchr.c \ upstream-freebsd/lib/libc/string/wmemcmp.c \ upstream-freebsd/lib/libc/string/wmemmove.c \ - upstream-openbsd/lib/libc/string/stpcpy.c \ libc_openbsd_src_files_arm += \ upstream-openbsd/lib/libc/string/stpncpy.c \ diff --git a/libc/arch-arm/cortex-a53/cortex-a53.mk b/libc/arch-arm/cortex-a53/cortex-a53.mk index c9ab31f42..e6fc66d6e 100644 --- a/libc/arch-arm/cortex-a53/cortex-a53.mk +++ b/libc/arch-arm/cortex-a53/cortex-a53.mk @@ -6,6 +6,7 @@ libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/strcmp.S \ arch-arm/cortex-a15/bionic/strcpy.S \ arch-arm/cortex-a15/bionic/strlen.S \ + arch-arm/cortex-a15/bionic/stpcpy.S \ arch-arm/cortex-a15/bionic/__strcat_chk.S \ arch-arm/cortex-a15/bionic/__strcpy_chk.S \ arch-arm/krait/bionic/memmove.S From 0073685d9175f88f7f416220c1e7798c295fb53f Mon Sep 17 00:00:00 2001 From: Danny Baumann Date: Thu, 26 Mar 2015 12:56:05 +0100 Subject: [PATCH 40/89] Fix mismerges. Change-Id: I88d0cc9d5396145c32940c0e2e3c3647df9782ee --- libc/arch-arm/arm.mk | 1 + libc/arch-arm/krait/bionic/memcpy_base.S | 87 ------------------------ 2 files changed, 1 insertion(+), 87 deletions(-) diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index ab82ec5f9..28f231fa4 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -74,6 +74,7 @@ ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),krait) ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),scorpion) ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),cortex-a53) libc_bionic_src_files_arm += \ + upstream-openbsd/lib/libc/string/stpcpy.c \ upstream-openbsd/lib/libc/string/bcopy.c endif endif diff --git a/libc/arch-arm/krait/bionic/memcpy_base.S b/libc/arch-arm/krait/bionic/memcpy_base.S index d95f9a113..068f2f60c 100644 --- a/libc/arch-arm/krait/bionic/memcpy_base.S +++ b/libc/arch-arm/krait/bionic/memcpy_base.S @@ -211,92 +211,5 @@ MEMCPY_BASE_ALIGNED: pop {r0, lr} bx lr //.cfi_endproc -// Assumes neon instructions and a cache line size of 32 bytes. - -ENTRY_PRIVATE(MEMCPY_BASE) - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - - /* do we have at least 16-bytes to copy (needed for alignment below) */ - cmp r2, #16 - blo 5f - - /* align destination to cache-line for the write-buffer */ - rsb r3, r0, #0 - ands r3, r3, #0xF - beq 2f - - /* copy up to 15-bytes (count in r3) */ - sub r2, r2, r3 - movs ip, r3, lsl #31 - itt mi - ldrbmi lr, [r1], #1 - strbmi lr, [r0], #1 - itttt cs - ldrbcs ip, [r1], #1 - ldrbcs lr, [r1], #1 - strbcs ip, [r0], #1 - strbcs lr, [r0], #1 - movs ip, r3, lsl #29 - bge 1f - // copies 4 bytes, destination 32-bits aligned - vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! - vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! -1: bcc 2f - // copies 8 bytes, destination 64-bits aligned - vld1.8 {d0}, [r1]! - vst1.8 {d0}, [r0, :64]! - -2: /* make sure we have at least 64 bytes to copy */ - subs r2, r2, #64 - blo 2f - -1: /* The main loop copies 64 bytes at a time */ - vld1.8 {d0 - d3}, [r1]! - vld1.8 {d4 - d7}, [r1]! - pld [r1, #(32*8)] - subs r2, r2, #64 - vst1.8 {d0 - d3}, [r0, :128]! - vst1.8 {d4 - d7}, [r0, :128]! - bhs 1b - -2: /* fix-up the remaining count and make sure we have >= 32 bytes left */ - adds r2, r2, #32 - blo 4f - - /* Copy 32 bytes. These cache lines were already preloaded */ - vld1.8 {d0 - d3}, [r1]! - sub r2, r2, #32 - vst1.8 {d0 - d3}, [r0, :128]! - -4: /* less than 32 left */ - add r2, r2, #32 - tst r2, #0x10 - beq 5f - // copies 16 bytes, 128-bits aligned - vld1.8 {d0, d1}, [r1]! - vst1.8 {d0, d1}, [r0, :128]! - -5: /* copy up to 15-bytes (count in r2) */ - movs ip, r2, lsl #29 - bcc 1f - vld1.8 {d0}, [r1]! - vst1.8 {d0}, [r0]! -1: bge 2f - vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! - vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! -2: movs ip, r2, lsl #31 - itt mi - ldrbmi r3, [r1], #1 - strbmi r3, [r0], #1 - itttt cs - ldrbcs ip, [r1], #1 - ldrbcs lr, [r1], #1 - strbcs ip, [r0], #1 - strbcs lr, [r0], #1 - - ldmfd sp!, {r0, lr} - bx lr END(MEMCPY_BASE) From 418f200c00b1695e2e1d3ceefebd5472b7bd8367 Mon Sep 17 00:00:00 2001 From: HashBang Date: Fri, 27 Mar 2015 18:46:26 -0400 Subject: [PATCH 41/89] Denver: fix duplicate of stpcpy.c Change-Id: I08cdfc1d22d4c0c98f3e5899089bde5c8ec3b9ba --- libc/arch-arm/arm.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index 28f231fa4..1040393b6 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -73,12 +73,14 @@ endif ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),krait) ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),scorpion) ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),cortex-a53) +ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),denver) libc_bionic_src_files_arm += \ upstream-openbsd/lib/libc/string/stpcpy.c \ upstream-openbsd/lib/libc/string/bcopy.c endif endif endif +endif cpu_variant_mk := $(LOCAL_PATH)/arch-arm/$(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)/$(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT).mk ifeq ($(wildcard $(cpu_variant_mk)),) $(error "TARGET_$(my_2nd_arch_prefix)CPU_VARIANT not set or set to an unknown value. Possible values are cortex-a7, cortex-a8, cortex-a9, cortex-a15, krait, scorpion, denver. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.") From 9ff06734f7a2100d99017de256e0ebfe4d16625d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=2E=20Andrei=20M=C4=83ce=C8=99?= Date: Sun, 29 Mar 2015 12:15:27 -0400 Subject: [PATCH 42/89] Cortex-A9: fix duplicate stpcpy Change-Id: I328a949f8a9ca8bbc9f00f6921215b618c7a5462 --- libc/arch-arm/arm.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index 1040393b6..e60138334 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -73,6 +73,7 @@ endif ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),krait) ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),scorpion) ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),cortex-a53) +ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),cortex-a9) ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),denver) libc_bionic_src_files_arm += \ upstream-openbsd/lib/libc/string/stpcpy.c \ @@ -81,6 +82,7 @@ endif endif endif endif +endif cpu_variant_mk := $(LOCAL_PATH)/arch-arm/$(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)/$(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT).mk ifeq ($(wildcard $(cpu_variant_mk)),) $(error "TARGET_$(my_2nd_arch_prefix)CPU_VARIANT not set or set to an unknown value. Possible values are cortex-a7, cortex-a8, cortex-a9, cortex-a15, krait, scorpion, denver. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.") From 8a96ea662ae987f3b6b40d023b388848b0cce91d Mon Sep 17 00:00:00 2001 From: Utkarsh Gupta Date: Thu, 2 Apr 2015 11:11:11 +0530 Subject: [PATCH 43/89] libc: Filter stpcpy and bcopy by architecture. Change-Id: I0e3a90d4d42e99082e1544f321e33f23d6b4998c --- libc/arch-arm/arm.mk | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index e60138334..36612de9a 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -70,19 +70,17 @@ libc_arch_dynamic_src_files_arm := arch-arm/bionic/exidx_dynamic.c ifeq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),) $(warning TARGET_$(my_2nd_arch_prefix)ARCH is arm, but TARGET_$(my_2nd_arch_prefix)CPU_VARIANT is not defined) endif -ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),krait) -ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),scorpion) -ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),cortex-a53) -ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),cortex-a9) -ifneq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),denver) + +ifeq ($(filter cortex-a9 cortex-a53 denver krait scorpion, $(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT))),) libc_bionic_src_files_arm += \ - upstream-openbsd/lib/libc/string/stpcpy.c \ - upstream-openbsd/lib/libc/string/bcopy.c -endif -endif -endif + upstream-openbsd/lib/libc/string/stpcpy.c endif + +ifeq ($(filter cortex-a53 denver krait scorpion, $(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT))),) +libc_bionic_src_files_arm += \ + upstream-openbsd/lib/libc/string/bcopy.c endif + cpu_variant_mk := $(LOCAL_PATH)/arch-arm/$(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)/$(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT).mk ifeq ($(wildcard $(cpu_variant_mk)),) $(error "TARGET_$(my_2nd_arch_prefix)CPU_VARIANT not set or set to an unknown value. Possible values are cortex-a7, cortex-a8, cortex-a9, cortex-a15, krait, scorpion, denver. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.") From 6bb134bfc332c9c7f800e4369d284e0298c55f95 Mon Sep 17 00:00:00 2001 From: HashBang Date: Tue, 12 May 2015 17:08:22 -0400 Subject: [PATCH 44/89] libc: denver should compile bcopy Change-Id: Id3c9a9c1a32a9026917c74cd4e8594c0e9c711e0 --- libc/arch-arm/arm.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index 36612de9a..2066c22ed 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -76,7 +76,7 @@ libc_bionic_src_files_arm += \ upstream-openbsd/lib/libc/string/stpcpy.c endif -ifeq ($(filter cortex-a53 denver krait scorpion, $(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT))),) +ifeq ($(filter cortex-a53 krait scorpion, $(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT))),) libc_bionic_src_files_arm += \ upstream-openbsd/lib/libc/string/bcopy.c endif From 8b8e5d32202bb15a93fa081bcaf18e1ec5dbe326 Mon Sep 17 00:00:00 2001 From: Varun Chitre Date: Fri, 22 May 2015 05:56:47 -0400 Subject: [PATCH 45/89] libc : memcpy : allow to disable opt for some targets Change-Id: I487b9e208c032c7d01523e9b36984075027e9bff --- libc/arch-arm/cortex-a15/cortex-a15.mk | 10 +- libc/arch-arm/cortex-a7/bionic/memcpy.S | 103 ++++++ libc/arch-arm/cortex-a7/bionic/memcpy_base.S | 318 +++++++++++++++++++ 3 files changed, 430 insertions(+), 1 deletion(-) create mode 100644 libc/arch-arm/cortex-a7/bionic/memcpy.S create mode 100644 libc/arch-arm/cortex-a7/bionic/memcpy_base.S diff --git a/libc/arch-arm/cortex-a15/cortex-a15.mk b/libc/arch-arm/cortex-a15/cortex-a15.mk index ac5fbbe57..b4279f802 100644 --- a/libc/arch-arm/cortex-a15/cortex-a15.mk +++ b/libc/arch-arm/cortex-a15/cortex-a15.mk @@ -1,6 +1,5 @@ libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/memchr.S \ - arch-arm/cortex-a15/bionic/memcpy.S \ arch-arm/cortex-a15/bionic/memset.S \ arch-arm/cortex-a15/bionic/strcat.S \ arch-arm/cortex-a15/bionic/__strcat_chk.S \ @@ -9,3 +8,12 @@ libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/__strcpy_chk.S \ arch-arm/cortex-a15/bionic/strlen.S \ bionic/memmove.c \ + +# Optimization not required for some targets +ifeq ($(TARGET_CPU_MEMCPY_OPT_DISABLE),true) +libc_bionic_src_files_arm += \ + arch-arm/cortex-a7/bionic/memcpy.S +else +libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memcpy.S +endif diff --git a/libc/arch-arm/cortex-a7/bionic/memcpy.S b/libc/arch-arm/cortex-a7/bionic/memcpy.S new file mode 100644 index 000000000..410b663e2 --- /dev/null +++ b/libc/arch-arm/cortex-a7/bionic/memcpy.S @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// Prototype: void *memcpy (void *dst, const void *src, size_t count). + +#include +#include + + .text + .syntax unified + .fpu neon + +ENTRY(__memcpy_chk) + cmp r2, r3 + bhi __memcpy_chk_fail + + // Fall through to memcpy... +END(__memcpy_chk) + +ENTRY(memcpy) + pld [r1, #64] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 +END(memcpy) + +#define MEMCPY_BASE __memcpy_base +#define MEMCPY_BASE_ALIGNED __memcpy_base_aligned +#include "memcpy_base.S" + +ENTRY_PRIVATE(__memcpy_chk_fail) + // Preserve lr for backtrace. + push {lr} + .cfi_def_cfa_offset 4 + .cfi_rel_offset lr, 0 + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) +END(__memcpy_chk_fail) + + .data +error_string: + .string "memcpy: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a7/bionic/memcpy_base.S b/libc/arch-arm/cortex-a7/bionic/memcpy_base.S new file mode 100644 index 000000000..2a7385247 --- /dev/null +++ b/libc/arch-arm/cortex-a7/bionic/memcpy_base.S @@ -0,0 +1,318 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +ENTRY_PRIVATE(MEMCPY_BASE) + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + + // Assumes that n >= 0, and dst, src are valid pointers. + // For any sizes less than 832 use the neon code that doesn't + // care about the src alignment. This avoids any checks + // for src alignment, and offers the best improvement since + // smaller sized copies are dominated by the overhead of + // the pre and post main loop. + // For larger copies, if src and dst cannot both be aligned to + // word boundaries, use the neon code. + // For all other copies, align dst to a double word boundary + // and copy using LDRD/STRD instructions. + + cmp r2, #16 + blo .L_copy_less_than_16_unknown_align + + // TODO: The aligned copy code is extremely slow copying some large + // buffers so always go through the unaligned path for now. + //cmp r2, #832 + //bge .L_check_alignment + +.L_copy_unknown_alignment: + // Unknown alignment of src and dst. + // Assumes that the first few bytes have already been prefetched. + + // Align destination to 128 bits. The mainloop store instructions + // require this alignment or they will throw an exception. + rsb r3, r0, #0 + ands r3, r3, #0xF + beq 2f + + // Copy up to 15 bytes (count in r3). + sub r2, r2, r3 + movs ip, r3, lsl #31 + + itt mi + ldrbmi lr, [r1], #1 + strbmi lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1], #1 + strbcs ip, [r0], #1 + strbcs lr, [r0], #1 + + movs ip, r3, lsl #29 + bge 1f + // Copies 4 bytes, dst 32 bits aligned before, at least 64 bits after. + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! +1: bcc 2f + // Copies 8 bytes, dst 64 bits aligned before, at least 128 bits after. + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0, :64]! + +2: // Make sure we have at least 64 bytes to copy. + subs r2, r2, #64 + blo 2f + +1: // The main loop copies 64 bytes at a time. + vld1.8 {d0 - d3}, [r1]! + vld1.8 {d4 - d7}, [r1]! + pld [r1, #(64*4)] + subs r2, r2, #64 + vst1.8 {d0 - d3}, [r0, :128]! + vst1.8 {d4 - d7}, [r0, :128]! + bhs 1b + +2: // Fix-up the remaining count and make sure we have >= 32 bytes left. + adds r2, r2, #32 + blo 3f + + // 32 bytes. These cache lines were already preloaded. + vld1.8 {d0 - d3}, [r1]! + sub r2, r2, #32 + vst1.8 {d0 - d3}, [r0, :128]! +3: // Less than 32 left. + add r2, r2, #32 + tst r2, #0x10 + beq .L_copy_less_than_16_unknown_align + // Copies 16 bytes, destination 128 bits aligned. + vld1.8 {d0, d1}, [r1]! + vst1.8 {d0, d1}, [r0, :128]! + +.L_copy_less_than_16_unknown_align: + // Copy up to 15 bytes (count in r2). + movs ip, r2, lsl #29 + bcc 1f + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0]! +1: bge 2f + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! + +2: // Copy 0 to 4 bytes. + lsls r2, r2, #31 + itt ne + ldrbne lr, [r1], #1 + strbne lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1] + strbcs ip, [r0], #1 + strbcs lr, [r0] + + pop {r0, pc} + +.L_check_alignment: + // If src and dst cannot both be aligned to a word boundary, + // use the unaligned copy version. + eor r3, r0, r1 + ands r3, r3, #0x3 + bne .L_copy_unknown_alignment +END(MEMCPY_BASE) + +ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED) + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + + // To try and improve performance, stack layout changed, + // i.e., not keeping the stack looking like users expect + // (highest numbered register at highest address). + strd r4, r5, [sp, #-8]! + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 + strd r6, r7, [sp, #-8]! + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r6, 0 + .cfi_rel_offset r7, 0 + strd r8, r9, [sp, #-8]! + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r8, 0 + .cfi_rel_offset r9, 4 + + // Optimized for already aligned dst code. + ands ip, r0, #3 + bne .L_dst_not_word_aligned + +.L_word_aligned: + // Align the destination buffer to 8 bytes, to make sure double + // loads and stores don't cross a cache line boundary, + // as they are then more expensive even if the data is in the cache + // (require two load/store issue cycles instead of one). + // If only one of the buffers is not 8 bytes aligned, + // then it's more important to align dst than src, + // because there is more penalty for stores + // than loads that cross a cacheline boundary. + // This check and realignment are only done if there is >= 832 + // bytes to copy. + + // Dst is word aligned, but check if it is already double word aligned. + ands r3, r0, #4 + beq 1f + ldr r3, [r1], #4 + str r3, [r0], #4 + sub r2, #4 + +1: // Can only get here if > 64 bytes to copy, so don't do check r2. + sub r2, #64 + +2: // Every loop iteration copies 64 bytes. + .irp offset, #0, #8, #16, #24, #32 + ldrd r4, r5, [r1, \offset] + strd r4, r5, [r0, \offset] + .endr + + ldrd r4, r5, [r1, #40] + ldrd r6, r7, [r1, #48] + ldrd r8, r9, [r1, #56] + + // Keep the pld as far from the next load as possible. + // The amount to prefetch was determined experimentally using + // large sizes, and verifying the prefetch size does not affect + // the smaller copies too much. + // WARNING: If the ldrd and strd instructions get too far away + // from each other, performance suffers. Three loads + // in a row is the best tradeoff. + pld [r1, #(64*16)] + strd r4, r5, [r0, #40] + strd r6, r7, [r0, #48] + strd r8, r9, [r0, #56] + + add r0, r0, #64 + add r1, r1, #64 + subs r2, r2, #64 + bge 2b + + // Fix-up the remaining count and make sure we have >= 32 bytes left. + adds r2, r2, #32 + blo 4f + + // Copy 32 bytes. These cache lines were already preloaded. + .irp offset, #0, #8, #16, #24 + ldrd r4, r5, [r1, \offset] + strd r4, r5, [r0, \offset] + .endr + add r1, r1, #32 + add r0, r0, #32 + sub r2, r2, #32 +4: // Less than 32 left. + add r2, r2, #32 + tst r2, #0x10 + beq 5f + // Copy 16 bytes. + .irp offset, #0, #8 + ldrd r4, r5, [r1, \offset] + strd r4, r5, [r0, \offset] + .endr + add r1, r1, #16 + add r0, r0, #16 + +5: // Copy up to 15 bytes (count in r2). + movs ip, r2, lsl #29 + bcc 1f + // Copy 8 bytes. + ldrd r4, r5, [r1], #8 + strd r4, r5, [r0], #8 +1: bge 2f + // Copy 4 bytes. + ldr r4, [r1], #4 + str r4, [r0], #4 +2: // Copy 0 to 4 bytes. + lsls r2, r2, #31 + itt ne + ldrbne lr, [r1], #1 + strbne lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1] + strbcs ip, [r0], #1 + strbcs lr, [r0] + + // Restore registers: optimized pop {r0, pc} + ldrd r8, r9, [sp], #8 + ldrd r6, r7, [sp], #8 + ldrd r4, r5, [sp], #8 + pop {r0, pc} + +.L_dst_not_word_aligned: + // Align dst to word. + rsb ip, ip, #4 + cmp ip, #2 + + itt gt + ldrbgt lr, [r1], #1 + strbgt lr, [r0], #1 + + itt ge + ldrbge lr, [r1], #1 + strbge lr, [r0], #1 + + ldrb lr, [r1], #1 + strb lr, [r0], #1 + + sub r2, r2, ip + + // Src is guaranteed to be at least word aligned by this point. + b .L_word_aligned +END(MEMCPY_BASE_ALIGNED) From ccab44cf3461727823c525a50657aaf7c69026ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= Date: Fri, 24 Apr 2015 16:21:38 +0200 Subject: [PATCH 46/89] Define char16_t and char32_t to make gcc 5.1 happy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gcc 5.1 doesn't define char16_t and char32_t (unless in C++ mode), causing compile failures. Change-Id: I08dcd13cdf8cd59a4a2f191864bedf4c0d1bb313 Signed-off-by: Bernhard Rosenkränzer Signed-off-by: W4TCH0UT --- libc/include/uchar.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libc/include/uchar.h b/libc/include/uchar.h index e1fcb5c65..0ec9d2eb5 100644 --- a/libc/include/uchar.h +++ b/libc/include/uchar.h @@ -34,6 +34,11 @@ __BEGIN_DECLS +#if defined(__GNUC__) && __GNUC__ >= 5 && !defined(__cplusplus) +typedef __CHAR16_TYPE__ char16_t; +typedef __CHAR32_TYPE__ char32_t; +#endif + #define __STD_UTF_16__ 1 #define __STD_UTF_32__ 1 From ddbb10dd4497582d036fc0d33b12750640ced693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= Date: Sun, 30 Nov 2014 22:17:30 +0100 Subject: [PATCH 47/89] Specify .cpu cortex-a15 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When building with clang without this change, as errors out saying pldw is an unsupported instruction (because it isn't part of the ARMv7 core instruction set). Let as know using pldw is fine. Change-Id: Ie1f9c4b873e93ab2b3b374d2d46e476a4e581710 Signed-off-by: Bernhard Rosenkränzer Signed-off-by: W4TCH0UT --- libc/arch-arm/denver/bionic/memset.S | 1 + 1 file changed, 1 insertion(+) diff --git a/libc/arch-arm/denver/bionic/memset.S b/libc/arch-arm/denver/bionic/memset.S index bf3d9aded..d77c2443b 100644 --- a/libc/arch-arm/denver/bionic/memset.S +++ b/libc/arch-arm/denver/bionic/memset.S @@ -37,6 +37,7 @@ * memset() returns its first argument. */ + .cpu cortex-a15 .fpu neon .syntax unified From 6442df9f9db3df148702953f3a9e81713e1ee67e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= Date: Thu, 12 Mar 2015 10:17:30 +0100 Subject: [PATCH 48/89] Fix build with clang MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix build with clang by adding missing initializations. This is already accepted on master. Change-Id: Ice43ff41f502c45222883dd28f861e511fbd0b97 Signed-off-by: Bernhard Rosenkränzer Signed-off-by: W4TCH0UT --- libc/stdio/findfp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libc/stdio/findfp.c b/libc/stdio/findfp.c index cfbb66bb8..7a537c8fe 100644 --- a/libc/stdio/findfp.c +++ b/libc/stdio/findfp.c @@ -53,8 +53,8 @@ int __sdidinit; #define NDYNAMIC 10 /* add ten more whenever necessary */ #define std(flags, file) \ - {0,0,0,flags,file,{0},0,__sF+file,__sclose,__sread,__sseek,__swrite, \ - {(unsigned char *)(__sFext+file), 0},NULL,0,{0},{0},{0},0,0} + {0,0,0,flags,file,{0,0},0,__sF+file,__sclose,__sread,__sseek,__swrite, \ + {(unsigned char *)(__sFext+file), 0},NULL,0,{0},{0},{0,0},0,0} /* the usual - (stdin + stdout + stderr) */ static FILE usual[FOPEN_MAX - 3]; From 4198b88f2b7212e61a00430ef12724eb0ce35501 Mon Sep 17 00:00:00 2001 From: Yingshiuan Pan Date: Fri, 6 Mar 2015 15:24:22 +0000 Subject: [PATCH 49/89] libc: arm64: add optimized strcpy implementation This optimization is extracted from cortex-strings and bionic-ized. stringbench result: |---------------+--------+--------+-------------+ | item | aosp | cortex | cortex gain | |---------------+--------+--------+-------------+ | strcpy 15000b | 5.23 s | 5.20s | 0.46% | |---------------+--------+--------+-------------+ | strcpy 15b | 3.64 s | 3.28s | 9.91% | |---------------+--------+--------+-------------+ Change-Id: If04ebf677f8f41b4555b00709d41cc09b11cc927 Signed-off-by: Yingshiuan Pan Signed-off-by: W4TCH0UT --- libc/arch-arm64/generic/bionic/string_copy.S | 461 +++++++++++-------- 1 file changed, 277 insertions(+), 184 deletions(-) diff --git a/libc/arch-arm64/generic/bionic/string_copy.S b/libc/arch-arm64/generic/bionic/string_copy.S index 2bf969d60..2691ca56a 100644 --- a/libc/arch-arm64/generic/bionic/string_copy.S +++ b/libc/arch-arm64/generic/bionic/string_copy.S @@ -1,34 +1,9 @@ /* - * Copyright (C) 2014 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ + strcpy/stpcpy - copy a string returning pointer to start/end. -/* - Copyright (c) 2014, Linaro Limited - All rights reserved. + Copyright (c) 2013, 2014, 2015 ARM Ltd. + All Rights Reserved. + Copyright (c) 2015, Linaro Ltd. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -37,9 +12,9 @@ * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of the Linaro nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. + * Neither the name of the company nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT @@ -51,12 +26,11 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Assumptions: * - * ARMv8-a, AArch64 + * ARMv8-a, AArch64, unaligned accesses, min page size 4k. */ #if !defined(STPCPY) && !defined(STRCPY) @@ -65,179 +39,298 @@ #include +/* To build as stpcpy, define STPCPY before compiling this file. + + To test the page crossing code path more thoroughly, compile with + -DSTRCPY_TEST_PAGE_CROSS - this will force all copies through the slower + entry path. This option is not intended for production use. */ + /* Arguments and results. */ -#if defined(STPCPY) -#define dst x0 -#elif defined(STRCPY) -#define dstin x0 -#endif -#define src x1 +#define dstin x0 +#define srcin x1 /* Locals and temporaries. */ -#if defined(STRCPY) -#define dst x2 -#endif -#define data1 x3 -#define data1_w w3 -#define data2 x4 -#define data2_w w4 -#define has_nul1 x5 -#define has_nul1_w w5 -#define has_nul2 x6 -#define tmp1 x7 -#define tmp2 x8 -#define tmp3 x9 -#define tmp4 x10 -#define zeroones x11 -#define zeroones_w w11 -#define pos x12 +#define src x2 +#define dst x3 +#define data1 x4 +#define data1w w4 +#define data2 x5 +#define data2w w5 +#define has_nul1 x6 +#define has_nul2 x7 +#define tmp1 x8 +#define tmp2 x9 +#define tmp3 x10 +#define tmp4 x11 +#define zeroones x12 +#define data1a x13 +#define data2a x14 +#define pos x15 +#define len x16 +#define to_align x17 + + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and + can be done in parallel across the entire word. */ #define REP8_01 0x0101010101010101 #define REP8_7f 0x7f7f7f7f7f7f7f7f #define REP8_80 0x8080808080808080 + /* AArch64 systems have a minimum page size of 4k. We can do a quick + page size check for crossing this boundary on entry and if we + do not, then we can short-circuit much of the entry code. We + expect early page-crossing strings to be rare (probability of + 16/MIN_PAGE_SIZE ~= 0.4%), so the branch should be quite + predictable, even with random strings. + + We don't bother checking for larger page sizes, the cost of setting + up the correct page size is just not worth the extra gain from + a small reduction in the cases taking the slow path. Note that + we only care about whether the first fetch, which may be + misaligned, crosses a page boundary - after that we move to aligned + fetches for the remainder of the string. */ + +#ifdef STRCPY_TEST_PAGE_CROSS + /* Make everything that isn't Qword aligned look like a page cross. */ +#define MIN_PAGE_P2 4 +#else +#define MIN_PAGE_P2 12 +#endif + +#define MIN_PAGE_SIZE (1 << MIN_PAGE_P2) + #if defined(STPCPY) ENTRY(stpcpy) #elif defined(STRCPY) ENTRY(strcpy) #endif - mov zeroones, #REP8_01 -#if defined(STRCPY) - mov dst, dstin + .p2align 6 + /* For moderately short strings, the fastest way to do the copy is to + calculate the length of the string in the same way as strlen, then + essentially do a memcpy of the result. This avoids the need for + multiple byte copies and further means that by the time we + reach the bulk copy loop we know we can always use DWord + accesses. We expect strcpy to rarely be called repeatedly + with the same source string, so branch prediction is likely to + always be difficult - we mitigate against this by preferring + conditional select operations over branches whenever this is + feasible. */ + and tmp2, srcin, #(MIN_PAGE_SIZE - 1) + mov zeroones, #REP8_01 + and to_align, srcin, #15 + cmp tmp2, #(MIN_PAGE_SIZE - 16) + neg tmp1, to_align + /* The first fetch will straddle a (possible) page boundary iff + srcin + 15 causes bit[MIN_PAGE_P2] to change value. A 16-byte + aligned string will never fail the page align check, so will + always take the fast path. */ + b.gt .Lpage_cross + +.Lpage_cross_ok: + ldp data1, data2, [srcin] +#ifdef __AARCH64EB__ + /* Because we expect the end to be found within 16 characters + (profiling shows this is the most common case), it's worth + swapping the bytes now to save having to recalculate the + termination syndrome later. We preserve data1 and data2 + so that we can re-use the values later on. */ + rev tmp2, data1 + sub tmp1, tmp2, zeroones + orr tmp2, tmp2, #REP8_7f + bics has_nul1, tmp1, tmp2 + b.ne .Lfp_le8 + rev tmp4, data2 + sub tmp3, tmp4, zeroones + orr tmp4, tmp4, #REP8_7f +#else + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + bics has_nul1, tmp1, tmp2 + b.ne .Lfp_le8 + sub tmp3, data2, zeroones + orr tmp4, data2, #REP8_7f #endif - ands tmp1, src, #15 - b.ne .Lmisaligned - // NUL detection works on the principle that (X - 1) & (~X) & 0x80 - // (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and - // can be done in parallel across the entire word. - // The inner loop deals with two Dwords at a time. This has a - // slightly higher start-up cost, but we should win quite quickly, - // especially on cores with a high number of issue slots per - // cycle, as we get much better parallelism out of the operations. -.Lloop: - ldp data1, data2, [src], #16 - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - bic has_nul1, tmp1, tmp2 - cbnz has_nul1, .Lnul_in_data1 - sub tmp3, data2, zeroones - orr tmp4, data2, #REP8_7f - bic has_nul2, tmp3, tmp4 - cbnz has_nul2, .Lnul_in_data2 - // No NUL in either register, copy it in a single instruction. - stp data1, data2, [dst], #16 - b .Lloop - -.Lnul_in_data1: - rev has_nul1, has_nul1 - clz pos, has_nul1 - add tmp1, pos, #0x8 - - tbz tmp1, #6, 1f -#if defined(STPCPY) - str data1, [dst], #7 -#elif defined(STRCPY) - str data1, [dst] + bics has_nul2, tmp3, tmp4 + b.eq .Lbulk_entry + + /* The string is short (<=16 bytes). We don't know exactly how + short though, yet. Work out the exact length so that we can + quickly select the optimal copy strategy. */ +.Lfp_gt8: + rev has_nul2, has_nul2 + clz pos, has_nul2 + mov tmp2, #56 + add dst, dstin, pos, lsr #3 /* Bits to bytes. */ + sub pos, tmp2, pos +#ifdef __AARCH64EB__ + lsr data2, data2, pos +#else + lsl data2, data2, pos #endif - ret -1: - tbz tmp1, #5, 1f - str data1_w, [dst], #4 - lsr data1, data1, #32 -1: - tbz tmp1, #4, 1f - strh data1_w, [dst], #2 - lsr data1, data1, #16 -1: - tbz tmp1, #3, 1f - strb data1_w, [dst] -#if defined(STPCPY) - ret + str data2, [dst, #1] + str data1, [dstin] +#ifdef STPCPY + add dstin, dst, #8 #endif -1: -#if defined(STPCPY) - // Back up one so that dst points to the '\0' string terminator. - sub dst, dst, #1 + ret + +.Lfp_le8: + rev has_nul1, has_nul1 + clz pos, has_nul1 + add dst, dstin, pos, lsr #3 /* Bits to bytes. */ + subs tmp2, pos, #24 /* Pos in bits. */ + b.lt .Lfp_lt4 +#ifdef __AARCH64EB__ + mov tmp2, #56 + sub pos, tmp2, pos + lsr data2, data1, pos + lsr data1, data1, #32 +#else + lsr data2, data1, tmp2 #endif - ret + /* 4->7 bytes to copy. */ + str data2w, [dst, #-3] + str data1w, [dstin] +#ifdef STPCPY + mov dstin, dst +#endif + ret +.Lfp_lt4: + cbz pos, .Lfp_lt2 + /* 2->3 bytes to copy. */ +#ifdef __AARCH64EB__ + lsr data1, data1, #48 +#endif + strh data1w, [dstin] + /* Fall-through, one byte (max) to go. */ +.Lfp_lt2: + /* Null-terminated string. Last character must be zero! */ + strb wzr, [dst] +#ifdef STPCPY + mov dstin, dst +#endif + ret -.Lnul_in_data2: - str data1, [dst], #8 - rev has_nul2, has_nul2 - clz pos, has_nul2 - add tmp1, pos, #0x8 + .p2align 6 + /* Aligning here ensures that the entry code and main loop all lies + within one 64-byte cache line. */ +.Lbulk_entry: + sub to_align, to_align, #16 + stp data1, data2, [dstin] + sub src, srcin, to_align + sub dst, dstin, to_align + b .Lentry_no_page_cross - tbz tmp1, #6, 1f -#if defined(STPCPY) - str data2, [dst], #7 -#elif defined(STRCPY) - str data2, [dst] -#endif - ret -1: - tbz tmp1, #5, 1f - str data2_w, [dst], #4 - lsr data2, data2, #32 -1: - tbz tmp1, #4, 1f - strh data2_w, [dst], #2 - lsr data2, data2, #16 -1: - tbz tmp1, #3, 1f - strb data2_w, [dst] -#if defined(STPCPY) - ret + /* The inner loop deals with two Dwords at a time. This has a + slightly higher start-up cost, but we should win quite quickly, + especially on cores with a high number of issue slots per + cycle, as we get much better parallelism out of the operations. */ +.Lmain_loop: + stp data1, data2, [dst], #16 +.Lentry_no_page_cross: + ldp data1, data2, [src], #16 + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + sub tmp3, data2, zeroones + orr tmp4, data2, #REP8_7f + bic has_nul1, tmp1, tmp2 + bics has_nul2, tmp3, tmp4 + ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */ + b.eq .Lmain_loop + + /* Since we know we are copying at least 16 bytes, the fastest way + to deal with the tail is to determine the location of the + trailing NUL, then (re)copy the 16 bytes leading up to that. */ + cmp has_nul1, #0 +#ifdef __AARCH64EB__ + /* For big-endian, carry propagation (if the final byte in the + string is 0x01) means we cannot use has_nul directly. The + easiest way to get the correct byte is to byte-swap the data + and calculate the syndrome a second time. */ + csel data1, data1, data2, ne + rev data1, data1 + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + bic has_nul1, tmp1, tmp2 +#else + csel has_nul1, has_nul1, has_nul2, ne #endif -1: -#if defined(STPCPY) - // Back up one so that dst points to the '\0' string terminator. - sub dst, dst, #1 + rev has_nul1, has_nul1 + clz pos, has_nul1 + add tmp1, pos, #72 + add pos, pos, #8 + csel pos, pos, tmp1, ne + add src, src, pos, lsr #3 + add dst, dst, pos, lsr #3 + ldp data1, data2, [src, #-32] + stp data1, data2, [dst, #-16] +#ifdef STPCPY + sub dstin, dst, #1 #endif - ret + ret -.Lmisaligned: - tbz src, #0, 1f - ldrb data1_w, [src], #1 - strb data1_w, [dst], #1 - cbnz data1_w, 1f -#if defined(STPCPY) - // Back up one so that dst points to the '\0' string terminator. - sub dst, dst, #1 +.Lpage_cross: + bic src, srcin, #15 + /* Start by loading two words at [srcin & ~15], then forcing the + bytes that precede srcin to 0xff. This means they never look + like termination bytes. */ + ldp data1, data2, [src] + lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */ + tst to_align, #7 + csetm tmp2, ne +#ifdef __AARCH64EB__ + lsl tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */ +#else + lsr tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */ #endif - ret -1: - tbz src, #1, 1f - ldrb data1_w, [src], #1 - strb data1_w, [dst], #1 - cbz data1_w, .Ldone - ldrb data2_w, [src], #1 - strb data2_w, [dst], #1 - cbnz data2_w, 1f -.Ldone: -#if defined(STPCPY) - // Back up one so that dst points to the '\0' string terminator. - sub dst, dst, #1 + orr data1, data1, tmp2 + orr data2a, data2, tmp2 + cmp to_align, #8 + csinv data1, data1, xzr, lt + csel data2, data2, data2a, lt + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + sub tmp3, data2, zeroones + orr tmp4, data2, #REP8_7f + bic has_nul1, tmp1, tmp2 + bics has_nul2, tmp3, tmp4 + ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */ + b.eq .Lpage_cross_ok + /* We now need to make data1 and data2 look like they've been + loaded directly from srcin. Do a rotate on the 128-bit value. */ + lsl tmp1, to_align, #3 /* Bytes->bits. */ + neg tmp2, to_align, lsl #3 +#ifdef __AARCH64EB__ + lsl data1a, data1, tmp1 + lsr tmp4, data2, tmp2 + lsl data2, data2, tmp1 + orr tmp4, tmp4, data1a + cmp to_align, #8 + csel data1, tmp4, data2, lt + rev tmp2, data1 + rev tmp4, data2 + sub tmp1, tmp2, zeroones + orr tmp2, tmp2, #REP8_7f + sub tmp3, tmp4, zeroones + orr tmp4, tmp4, #REP8_7f +#else + lsr data1a, data1, tmp1 + lsl tmp4, data2, tmp2 + lsr data2, data2, tmp1 + orr tmp4, tmp4, data1a + cmp to_align, #8 + csel data1, tmp4, data2, lt + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + sub tmp3, data2, zeroones + orr tmp4, data2, #REP8_7f #endif - ret -1: - tbz src, #2, 1f - ldr data1_w, [src], #4 - // Check for a zero. - sub has_nul1_w, data1_w, zeroones_w - bic has_nul1_w, has_nul1_w, data1_w - ands has_nul1_w, has_nul1_w, #0x80808080 - b.ne .Lnul_in_data1 - str data1_w, [dst], #4 -1: - tbz src, #3, .Lloop - ldr data1, [src], #8 - // Check for a zero. - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - bics has_nul1, tmp1, tmp2 - b.ne .Lnul_in_data1 - str data1, [dst], #8 - b .Lloop + bic has_nul1, tmp1, tmp2 + cbnz has_nul1, .Lfp_le8 + bic has_nul2, tmp3, tmp4 + b .Lfp_gt8 + #if defined(STPCPY) END(stpcpy) #elif defined(STRCPY) From a041683810b2b4dc53cc3564887074ba1452f5dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernhard=20Rosenkr=C3=A4nzer?= Date: Sat, 25 Apr 2015 02:55:34 +0200 Subject: [PATCH 50/89] Reduce -Wstrict-overflow level to 2 to make gcc 5.1.0 happy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With -Wstrict-overflow=3 or higher, gcc 5.1.0 screams at: bionic/linker/linker_phdr.cpp: In function 'int _phdr_table_set_load_prot(const Elf64_Phdr*, size_t, Elf64_Addr, int)': bionic/linker/linker_phdr.cpp:419:12: error: assuming pointer wraparound does not occur when comparing P +- C1 with P +- C2 [-Werror=strict-overflow] static int _phdr_table_set_load_prot(const ElfW(Phdr)* phdr_table, size ^ bionic/linker/linker_phdr.cpp: In function 'int phdr_table_protect_gnu_relro(const Elf64_Phdr*, size_t, Elf64_Addr)': bionic/linker/linker_phdr.cpp:543:5: error: assuming pointer wraparound does not occur when comparing P +- C1 with P +- C2 [-Werror=strict-overflow] int phdr_table_protect_gnu_relro(const ElfW(Phdr)* phdr_table, ^ bionic/linker/linker_phdr.cpp: In function 'int phdr_table_serialize_gnu_relro(const Elf64_Phdr*, size_t, Elf64_Addr, int)': bionic/linker/linker_phdr.cpp:560:5: error: assuming pointer wraparound does not occur when comparing P +- C1 with P +- C2 [-Werror=strict-overflow] int phdr_table_serialize_gnu_relro(const ElfW(Phdr)* phdr_table, ^ bionic/linker/linker_phdr.cpp: In function 'int phdr_table_map_gnu_relro(const Elf64_Phdr*, size_t, Elf64_Addr, int)': bionic/linker/linker_phdr.cpp:607:5: error: assuming pointer wraparound does not occur when comparing P +- C1 with P +- C2 [-Werror=strict-overflow] int phdr_table_map_gnu_relro(const ElfW(Phdr)* phdr_table, ^ bionic/linker/linker_phdr.cpp: In member function 'bool ElfReader::CheckPhdr(Elf64_Addr)': bionic/linker/linker_phdr.cpp:789:6: error: assuming pointer wraparound does not occur when comparing P +- C1 with P +- C2 [-Werror=strict-overflow] bool ElfReader::CheckPhdr(ElfW(Addr) loaded) { ^ bionic/linker/linker_phdr.cpp: In member function 'bool ElfReader::FindPhdr()': bionic/linker/linker_phdr.cpp:757:6: error: assuming pointer wraparound does not occur when comparing P +- C1 with P +- C2 [-Werror=strict-overflow] bool ElfReader::FindPhdr() { ^ cc1plus: all warnings being treated as errors Change-Id: Id109d2723bbc7708463d96d3a2ce0e13006ed35a Signed-off-by: Bernhard Rosenkränzer Signed-off-by: W4TCH0UT --- linker/Android.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linker/Android.mk b/linker/Android.mk index 4298032a4..52f3139cc 100644 --- a/linker/Android.mk +++ b/linker/Android.mk @@ -26,7 +26,7 @@ LOCAL_LDFLAGS := \ LOCAL_CFLAGS += \ -fno-stack-protector \ - -Wstrict-overflow=5 \ + -Wstrict-overflow=2 \ -fvisibility=hidden \ -Wall -Wextra -Wunused -Werror \ From 7f5a3e9939c3ec66ee7d6e7702bb22c92ca33ba8 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Thu, 30 Apr 2015 02:13:56 -0500 Subject: [PATCH 51/89] Speed-up ARM memcmp by leveraging NEON. Because NEON_UNALIGNED_ACCESS has never been defined, it has gone unused. This change changes it to check for __ARM_FEATURE_UNALIGNED, defined by GCC. This has improved speed by roughly 34% on my testcases. Also, memcmp checks if HAVE_32_BYTE_CACHE_LINE is defined and adjusts for it; this is true for Cortex-A9, so define it in cortex-a9 makefile. Change-Id: I3c76ce7fa2796872e0171d5502b0ebd6e2893339 Signed-off-by: W4TCH0UT --- libc/arch-arm/bionic/memcmp.S | 2 +- libc/arch-arm/cortex-a9/cortex-a9.mk | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/libc/arch-arm/bionic/memcmp.S b/libc/arch-arm/bionic/memcmp.S index 70a2a58ce..90817fb48 100644 --- a/libc/arch-arm/bionic/memcmp.S +++ b/libc/arch-arm/bionic/memcmp.S @@ -61,7 +61,7 @@ ENTRY(memcmp) * Neon optimization * Comparing 32 bytes at a time */ -#if defined(__ARM_NEON__) && defined(NEON_UNALIGNED_ACCESS) +#if defined(__ARM_NEON__) && defined(__ARM_FEATURE_UNALIGNED) subs r2, r2, #32 blo 3f diff --git a/libc/arch-arm/cortex-a9/cortex-a9.mk b/libc/arch-arm/cortex-a9/cortex-a9.mk index 5805ea86c..af1e7cfa9 100644 --- a/libc/arch-arm/cortex-a9/cortex-a9.mk +++ b/libc/arch-arm/cortex-a9/cortex-a9.mk @@ -10,3 +10,6 @@ libc_bionic_src_files_arm += \ arch-arm/cortex-a9/bionic/__strcpy_chk.S \ arch-arm/cortex-a9/bionic/strlen.S \ bionic/memmove.c \ + +libc_common_cflags_arm += \ + -DHAVE_32_BYTE_CACHE_LINE From 4a08e0b6f1f1312ab505280636916e493cac4113 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Thu, 30 Apr 2015 05:26:08 -0500 Subject: [PATCH 52/89] benchmarks: Add strchr benchmark. Not totally sure on this code, but it works. Change-Id: I421cc7b02f9a4ff15c82674abe50a2dc332c25ed Signed-off-by: W4TCH0UT --- benchmarks/string_benchmark.cpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/benchmarks/string_benchmark.cpp b/benchmarks/string_benchmark.cpp index 536e253fd..b8a07ccb7 100644 --- a/benchmarks/string_benchmark.cpp +++ b/benchmarks/string_benchmark.cpp @@ -110,3 +110,24 @@ static void BM_string_strlen(int iters, int nbytes) { delete[] s; } BENCHMARK(BM_string_strlen)->AT_COMMON_SIZES; + +static void BM_string_strchr(int iters, int nbytes) { + StopBenchmarkTiming(); + char* s = new char[nbytes]; + memset(s, 'x', nbytes); + s[nbytes - 1] = 0; + StartBenchmarkTiming(); + + volatile char* c = new char[1]; + volatile int d __attribute__((unused)) = 0; + for (int i = 0; i < iters; ++i) { + c = strchr(s, 'y'); + if (c != NULL) + d += (int)c[0]; + } + + StopBenchmarkTiming(); + SetBenchmarkBytesProcessed(int64_t(iters) * int64_t(nbytes)); + delete[] s; +} +BENCHMARK(BM_string_strchr)->AT_COMMON_SIZES; From 7138003fa91ec99c91f628c6f133ca5a5ac65e35 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Thu, 30 Apr 2015 05:29:07 -0500 Subject: [PATCH 53/89] libc: arm: Add asm strchr. This is about 40% faster on my testcase. Change-Id: Ibf0237633f10d9e915a030abad75d427901b775d Signed-off-by: W4TCH0UT --- libc/arch-arm/arm.mk | 2 +- libc/arch-arm/bionic/strchr.S | 78 +++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 libc/arch-arm/bionic/strchr.S diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index 2066c22ed..d44cb6d65 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -24,7 +24,6 @@ libc_openbsd_src_files_arm += \ libc_bionic_src_files_arm += \ bionic/memrchr.c \ - bionic/strchr.cpp \ bionic/strnlen.c \ bionic/strrchr.cpp \ @@ -62,6 +61,7 @@ libc_bionic_src_files_arm += \ arch-arm/bionic/setjmp.S \ arch-arm/bionic/sigsetjmp.S \ arch-arm/bionic/syscall.S \ + arch-arm/bionic/strchr.S \ libc_arch_static_src_files_arm := arch-arm/bionic/exidx_static.c libc_arch_dynamic_src_files_arm := arch-arm/bionic/exidx_dynamic.c diff --git a/libc/arch-arm/bionic/strchr.S b/libc/arch-arm/bionic/strchr.S new file mode 100644 index 000000000..433d2b038 --- /dev/null +++ b/libc/arch-arm/bionic/strchr.S @@ -0,0 +1,78 @@ +/* Copyright (c) 2010-2011, Linaro Limited + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Linaro Limited nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + Written by Dave Gilbert + + A very simple strchr routine, from benchmarks on A9 it's a bit faster than + the current version in eglibc (2.12.1-0ubuntu14 package) + I don't think doing a word at a time version is worth it since a lot + of strchr cases are very short anyway. + + */ + +#include +#include + + .syntax unified + .arch armv7-a + + .text + .thumb + +@ --------------------------------------------------------------------------- + + .thumb_func + .p2align 4,,15 +ENTRY(strchr) + @ r0 = start of string + @ r1 = character to match + @ returns NULL for no match, or a pointer to the match + and r1,r1, #255 + +1: + ldrb r2,[r0],#1 + cmp r2,r1 + cbz r2,10f + bne 1b + + @ We're here if it matched +5: + subs r0,r0,#1 + bx lr + +10: + @ We're here if we ran off the end + cmp r1, #0 @ Corner case - you're allowed to search for the nil and get a pointer to it + beq 5b @ A bit messy, if it's common we should branch at the start to a special loop + mov r0,#0 + bx lr +END(strchr) From 8e51242b4ab95f0c321f53dbd06c9f8a7eee6666 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Thu, 30 Apr 2015 05:30:49 -0500 Subject: [PATCH 54/89] libc: Avoid building some extra debug stuff. Since I'm using mmm bionic/libc/ for testing, this was being built. I don't want it built. Change-Id: Ia86b605719ffebfd4eb7bdfde10b158bab490554 Signed-off-by: W4TCH0UT --- libc/Android.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libc/Android.mk b/libc/Android.mk index fb99c81d1..bfa8e41f4 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -1087,6 +1087,7 @@ include $(BUILD_SHARED_LIBRARY) # the checks are available only in emulator and are implemeted in # libc_malloc_qemu_instrumented.so. ifneq ($(TARGET_BUILD_VARIANT),user) +ifneq ($(TARGET_BUILD_VARIANT),userdebug) # ======================================================== # libc_malloc_debug_leak.so @@ -1160,6 +1161,7 @@ LOCAL_MODULE_TAGS := eng debug $(eval $(call patch-up-arch-specific-flags,LOCAL_CFLAGS,libc_common_cflags)) include $(BUILD_SHARED_LIBRARY) +endif #!userdebug endif #!user # ======================================================== From 530177694a6fa96b9c874a362f2b10e59657972e Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Thu, 30 Apr 2015 06:22:09 -0500 Subject: [PATCH 55/89] benchmarks: Add strrchr benchmark. Change-Id: Ibc401bf2a3b2fdf3951d3f73348f2f8519e787f9 Signed-off-by: W4TCH0UT --- benchmarks/string_benchmark.cpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/benchmarks/string_benchmark.cpp b/benchmarks/string_benchmark.cpp index b8a07ccb7..9d58257f7 100644 --- a/benchmarks/string_benchmark.cpp +++ b/benchmarks/string_benchmark.cpp @@ -131,3 +131,24 @@ static void BM_string_strchr(int iters, int nbytes) { delete[] s; } BENCHMARK(BM_string_strchr)->AT_COMMON_SIZES; + +static void BM_string_strrchr(int iters, int nbytes) { + StopBenchmarkTiming(); + char* s = new char[nbytes]; + memset(s, 'x', nbytes); + s[nbytes - 1] = 0; + StartBenchmarkTiming(); + + volatile char* c = new char[1]; + volatile int d __attribute__((unused)) = 0; + for (int i = 0; i < iters; ++i) { + c = strrchr(s, 'y'); + if (c != NULL) + d += (int)c[0]; + } + + StopBenchmarkTiming(); + SetBenchmarkBytesProcessed(int64_t(iters) * int64_t(nbytes)); + delete[] s; +} +BENCHMARK(BM_string_strrchr)->AT_COMMON_SIZES; From 4f053b128f5ab6e15624b246fe4605750ad62acf Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Fri, 1 May 2015 01:16:47 -0500 Subject: [PATCH 56/89] benchmarks: Add strcmp/strncmp benchmarks. Change-Id: I72dcb0cb41b389dc9af1a67f3c246575000948f6 Signed-off-by: W4TCH0UT --- benchmarks/string_benchmark.cpp | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/benchmarks/string_benchmark.cpp b/benchmarks/string_benchmark.cpp index 9d58257f7..6af202b34 100644 --- a/benchmarks/string_benchmark.cpp +++ b/benchmarks/string_benchmark.cpp @@ -152,3 +152,45 @@ static void BM_string_strrchr(int iters, int nbytes) { delete[] s; } BENCHMARK(BM_string_strrchr)->AT_COMMON_SIZES; + +static void BM_string_strcmp(int iters, int nbytes) { + StopBenchmarkTiming(); + char* src = new char[nbytes]; char* dst = new char[nbytes]; + memset(src, 'x', nbytes); + memset(dst, 'x', nbytes); + src[nbytes - 1] = 0; + dst[nbytes - 1] = 0; + StartBenchmarkTiming(); + + volatile int c __attribute__((unused)) = 0; + for (int i = 0; i < iters; ++i) { + c += strcmp(dst, src); + } + + StopBenchmarkTiming(); + SetBenchmarkBytesProcessed(int64_t(iters) * int64_t(nbytes)); + delete[] src; + delete[] dst; +} +BENCHMARK(BM_string_strcmp)->AT_COMMON_SIZES; + +static void BM_string_strncmp(int iters, int nbytes) { + StopBenchmarkTiming(); + char* src = new char[nbytes]; char* dst = new char[nbytes]; + memset(src, 'x', nbytes); + memset(dst, 'x', nbytes); + src[nbytes - 1] = 0; + dst[nbytes - 1] = 0; + StartBenchmarkTiming(); + + volatile int c __attribute__((unused)) = 0; + for (int i = 0; i < iters; ++i) { + c += strncmp(dst, src, nbytes - 1); + } + + StopBenchmarkTiming(); + SetBenchmarkBytesProcessed(int64_t(iters) * int64_t(nbytes)); + delete[] src; + delete[] dst; +} +BENCHMARK(BM_string_strncmp)->AT_COMMON_SIZES; From bcc9620dc1d92e4640d1ce2274ec88c27cb77af7 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Fri, 1 May 2015 01:30:19 -0500 Subject: [PATCH 57/89] benchmarks: Add memchr/memrchr benchmarks. These aren't exactly 'accurate' but works in a relative sense. Change-Id: If736e616b74d6ea3bd36a1ecd626476a7adc2ad5 Signed-off-by: W4TCH0UT --- benchmarks/string_benchmark.cpp | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/benchmarks/string_benchmark.cpp b/benchmarks/string_benchmark.cpp index 6af202b34..6d7c251ad 100644 --- a/benchmarks/string_benchmark.cpp +++ b/benchmarks/string_benchmark.cpp @@ -194,3 +194,45 @@ static void BM_string_strncmp(int iters, int nbytes) { delete[] dst; } BENCHMARK(BM_string_strncmp)->AT_COMMON_SIZES; + +static void BM_string_memchr(int iters, int nbytes) { + StopBenchmarkTiming(); + char* s = new char[nbytes]; + memset(s, 'x', nbytes); + s[nbytes - 1] = 0; + StartBenchmarkTiming(); + + volatile void* c __attribute__((unused)) = NULL; + volatile int d __attribute__((unused)) = 0; + for (int i = 0; i < iters; ++i) { + c = memchr(s, 'y', nbytes - 1); + if (c != NULL) + d += (int)c; + } + + StopBenchmarkTiming(); + SetBenchmarkBytesProcessed(int64_t(iters) * int64_t(nbytes)); + delete[] s; +} +BENCHMARK(BM_string_memchr)->AT_COMMON_SIZES; + +static void BM_string_memrchr(int iters, int nbytes) { + StopBenchmarkTiming(); + char* s = new char[nbytes]; + memset(s, 'x', nbytes); + s[nbytes - 1] = 0; + StartBenchmarkTiming(); + + volatile void* c __attribute__((unused)) = NULL; + volatile int d __attribute__((unused)) = 0; + for (int i = 0; i < iters; ++i) { + c = memrchr(s, 'y', nbytes - 1); + if (c != NULL) + d += (int)c; + } + + StopBenchmarkTiming(); + SetBenchmarkBytesProcessed(int64_t(iters) * int64_t(nbytes)); + delete[] s; +} +BENCHMARK(BM_string_memrchr)->AT_COMMON_SIZES; From ed0df2efad1039dca17c2460b5abfff4a3e2f365 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Fri, 1 May 2015 01:52:22 -0500 Subject: [PATCH 58/89] libc: arm: Slight improvement to memchr. When this memchr was submitted to newlib years back, ldmia was changed to ldrd. It was suggested that it would be faster on A15, without a hit to A8/A9; in fact, my A9 testcase showed a measurable improvement with this change. Change-Id: I21fce7514706661a220fea953f5016cdfd795ed7 Signed-off-by: W4TCH0UT --- libc/arch-arm/cortex-a15/bionic/memchr.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/arch-arm/cortex-a15/bionic/memchr.S b/libc/arch-arm/cortex-a15/bionic/memchr.S index 784d44a4c..50deb6236 100644 --- a/libc/arch-arm/cortex-a15/bionic/memchr.S +++ b/libc/arch-arm/cortex-a15/bionic/memchr.S @@ -93,7 +93,7 @@ ENTRY(memchr) movs r3, #0 15: - ldmia r0!,{r5,r6} + ldrd r5,r6,[r0],#8 subs r4, r4, #8 eor r5,r5, r1 @ Get it so that r5,r6 have 00's where the bytes match the target eor r6,r6, r1 From 3d1cf15ca7bc6c7359c0bb1dd8d1b3156dd839d3 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Fri, 1 May 2015 07:21:38 -0500 Subject: [PATCH 59/89] cortex-a9: Weird improvement to strlen if we 'hit the lottery'. By using ldrd instead of ldmia in these places (yes, ALL of them), we get a crazy improvement to strlen in very specific cases (most notably, ~25% faster on 8 byte strings). Change-Id: Id92f3192e9f87597f5a07238aba09b688cd8d593 Signed-off-by: W4TCH0UT --- libc/arch-arm/cortex-a9/bionic/__strcat_chk.S | 2 +- libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S | 2 +- libc/arch-arm/cortex-a9/bionic/strcat.S | 2 +- libc/arch-arm/cortex-a9/bionic/string_copy.S | 2 +- libc/arch-arm/cortex-a9/bionic/strlen.S | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S index 45517f146..0ece9630f 100644 --- a/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S +++ b/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S @@ -65,7 +65,7 @@ ENTRY(__strcat_chk) .p2align 2 .L_mainloop: - ldmia r1!, {r2, r3} + ldrd r2, r3, [r1], #8 pld [r1, #64] diff --git a/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S index 67eca086e..562b68873 100644 --- a/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S +++ b/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S @@ -51,7 +51,7 @@ ENTRY(__strcpy_chk) .p2align 2 .L_mainloop: - ldmia r0!, {r2, r3} + ldrd r2, r3, [r0], #8 pld [r0, #64] diff --git a/libc/arch-arm/cortex-a9/bionic/strcat.S b/libc/arch-arm/cortex-a9/bionic/strcat.S index f5a855e39..6f7203284 100644 --- a/libc/arch-arm/cortex-a9/bionic/strcat.S +++ b/libc/arch-arm/cortex-a9/bionic/strcat.S @@ -164,7 +164,7 @@ strcpy_check_src_align: .p2align 2 strcpy_mainloop: - ldmia r1!, {r2, r3} + ldrd r2, r3, [r1], #8 pld [r1, #64] diff --git a/libc/arch-arm/cortex-a9/bionic/string_copy.S b/libc/arch-arm/cortex-a9/bionic/string_copy.S index caf5a11fe..f691a08cb 100644 --- a/libc/arch-arm/cortex-a9/bionic/string_copy.S +++ b/libc/arch-arm/cortex-a9/bionic/string_copy.S @@ -134,7 +134,7 @@ ENTRY(strcpy) .p2align 2 .Lstringcopy_mainloop: - ldmia r1!, {r2, r3} + ldrd r2, r3, [r1], #8 pld [r1, #64] diff --git a/libc/arch-arm/cortex-a9/bionic/strlen.S b/libc/arch-arm/cortex-a9/bionic/strlen.S index b92b04352..271bc29f4 100644 --- a/libc/arch-arm/cortex-a9/bionic/strlen.S +++ b/libc/arch-arm/cortex-a9/bionic/strlen.S @@ -69,7 +69,7 @@ ENTRY(strlen) .p2align 2 mainloop: - ldmia r1!, {r2, r3} + ldrd r2, r3, [r1], #8 pld [r1, #64] From dbb78debb7e151cccb2698d531ac8ebda339fdc6 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Fri, 1 May 2015 23:44:45 -0500 Subject: [PATCH 60/89] libc: arm: Remove a couple extra '.text's. ENTRY takes care of this for us. Change-Id: Ic00ea8fa23c2c64edea4353970c47f6b02abcb6e Signed-off-by: W4TCH0UT --- libc/arch-arm/bionic/strchr.S | 1 - libc/arch-arm/cortex-a15/bionic/memchr.S | 1 - 2 files changed, 2 deletions(-) diff --git a/libc/arch-arm/bionic/strchr.S b/libc/arch-arm/bionic/strchr.S index 433d2b038..423b14d71 100644 --- a/libc/arch-arm/bionic/strchr.S +++ b/libc/arch-arm/bionic/strchr.S @@ -45,7 +45,6 @@ .syntax unified .arch armv7-a - .text .thumb @ --------------------------------------------------------------------------- diff --git a/libc/arch-arm/cortex-a15/bionic/memchr.S b/libc/arch-arm/cortex-a15/bionic/memchr.S index 50deb6236..328fe81e8 100644 --- a/libc/arch-arm/cortex-a15/bionic/memchr.S +++ b/libc/arch-arm/cortex-a15/bionic/memchr.S @@ -54,7 +54,6 @@ #else #define CHARTSTMASK(c) 1<<(c*8) #endif - .text .thumb @ --------------------------------------------------------------------------- From b092454193862e5c220768ff798078e099a4a5a6 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Sat, 2 May 2015 00:53:16 -0500 Subject: [PATCH 61/89] libc: arm: Optimized strrchr courtesy of NetBSD. Change-Id: I295b371522336dd337a881ed0ca50f7f7f295582 Signed-off-by: W4TCH0UT --- libc/arch-arm/arm.mk | 2 +- libc/arch-arm/bionic/strrchr.S | 131 +++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 libc/arch-arm/bionic/strrchr.S diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index d44cb6d65..707e6479f 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -25,7 +25,6 @@ libc_openbsd_src_files_arm += \ libc_bionic_src_files_arm += \ bionic/memrchr.c \ bionic/strnlen.c \ - bionic/strrchr.cpp \ libc_freebsd_src_files_arm += \ upstream-freebsd/lib/libc/string/wcscat.c \ @@ -62,6 +61,7 @@ libc_bionic_src_files_arm += \ arch-arm/bionic/sigsetjmp.S \ arch-arm/bionic/syscall.S \ arch-arm/bionic/strchr.S \ + arch-arm/bionic/strrchr.S \ libc_arch_static_src_files_arm := arch-arm/bionic/exidx_static.c libc_arch_dynamic_src_files_arm := arch-arm/bionic/exidx_dynamic.c diff --git a/libc/arch-arm/bionic/strrchr.S b/libc/arch-arm/bionic/strrchr.S new file mode 100644 index 000000000..56d37d45c --- /dev/null +++ b/libc/arch-arm/bionic/strrchr.S @@ -0,0 +1,131 @@ +/*- + * Copyright (c) 2013 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Matt Thomas of 3am Software Foundry. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#ifndef __ARMEB__ +#define BYTE0 0x000000ff +#define BYTE1 0x0000ff00 +#define BYTE2 0x00ff0000 +#define BYTE3 0xff000000 +#define lshi lsl +#define lshis lsls +#else +#define BYTE0 0xff000000 +#define BYTE1 0x00ff0000 +#define BYTE2 0x0000ff00 +#define BYTE3 0x000000ff +#define lshi lsr +#define lshis lsrs +#endif + + .syntax unified + .arch armv7-a + + .thumb + .thumb_func + +ENTRY(strrchr) + ands r2, r1, #0xff /* is the byte value NUL? */ + bne 1f /* no, do it the hard way */ + push {r0, lr} /* save pointer and return addr */ + bl strlen /* get length */ + pop {r1, r2} /* restore pointer / return addr */ + adds r0, r0, r1 /* add pointer to length */ + bx r2 /* return */ + +1: mov r1, r0 /* we use r0 at the return value */ + movs r0, #0 /* return NULL by default */ +2: tst r1, #3 /* test for word alignment */ + beq .Lpre_main_loop /* finally word aligned */ + ldrb r3, [r1], #1 /* load a byte */ + cmp r3, r2 /* did it match? */ + it eq + subeq r0, r1, #1 /* yes, remember that it did */ + cmp r3, #0 /* was it NUL? */ + bne 2b /* no, try next byte */ + bx lr /* return */ +.Lpre_main_loop: + push {r4, r5} /* save some registers */ + movw ip, #0xfefe /* magic constant; 254 in each byte */ + movt ip, #0xfefe /* magic constant; 254 in each byte */ + orr r2, r2, r2, lsl #8 /* move to next byte */ + orr r2, r2, r2, lsl #16 /* move to next halfword */ +.Lmain_loop: + ldr r3, [r1], #4 /* load next word */ + /* + * Add 254 to each byte using the UQADD8 (unsigned saturating add 8) + * instruction. For every non-NUL byte, the result for that byte will + * become 255. For NUL, it will be 254. When we complement the + * result, if the result is non-0 then we must have encountered a NUL. + */ + uqadd8 r4, r3, ip /* NUL detection happens here */ + usub8 r3, r3, r2 /* bias for char looked for? */ + uqadd8 r5, r3, ip /* char detection happens here */ + ands r3, r4, r5 /* merge results */ + mvns r3, r3 /* is the complement non-0? */ + beq .Lmain_loop /* no, then keep going */ + + mvns r5, r5 /* get we find any matching bytes? */ + beq .Ldone /* no, then we hit the end, return */ + mvns r4, r4 /* did we encounter a NUL? */ + beq .Lfind_match /* no, find matching byte */ + /* + * Copy the NUL bit to the following byte lanes. Then clear any match + * bits in those byte lanes to prevent false positives in those bytes. + */ + bics r5, r5, r4 /* clear any NUL match bits */ + beq .Ldone /* no remaining matches, we're done */ + lshis r3, r4, #8 /* shift up a byte */ + itt ne + orrsne r3, r3, r3, lshi #8 /* if non 0, copy up to next byte */ + orrsne r3, r3, r3, lshi #8 /* if non 0, copy up to last byte */ + bics r5, r5, r3 /* clear match bits */ + beq .Ldone /* no remaining matches, we're done */ +.Lfind_match: +#ifndef __ARMEB__ + rev r5, r5 /* we want this in BE for the CLZ */ +#endif + /* + * If we have multiple matches, we want to the select the "last" match + * in the word which will be the lowest bit set. + */ + subs r3, r5, #1 /* subtract 1 */ + ands r3, r3, r5 /* and with mask */ + eors r5, r5, r3 /* only have the lowest bit set left */ + clz r5, r5 /* count how many leading zeros */ + add r0, r1, r5, lsr #3 /* divide that by 8 and add to count */ + subs r0, r0, #4 /* compensate for the post-inc */ + cmp r4, #0 /* did we read any NULs? */ + beq .Lmain_loop /* no, get next word */ +.Ldone: + pop {r4, r5} + bx lr +END(strrchr) From ce9ec0e88248dce44896552a103a749aaef7f54e Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Sat, 2 May 2015 01:13:20 -0500 Subject: [PATCH 62/89] libc: arm: Further optimized strchr, courtesy NetBSD. This is very competitive at the short cases, yet is nearly twice as fast at larger cases. Change-Id: Ic55e8414a02f7ad32ec08a06f4ae1750121d4f37 Signed-off-by: W4TCH0UT --- libc/arch-arm/bionic/strchr.S | 168 +++++++++++++++++++++------------- 1 file changed, 103 insertions(+), 65 deletions(-) diff --git a/libc/arch-arm/bionic/strchr.S b/libc/arch-arm/bionic/strchr.S index 423b14d71..75abf51b9 100644 --- a/libc/arch-arm/bionic/strchr.S +++ b/libc/arch-arm/bionic/strchr.S @@ -1,77 +1,115 @@ -/* Copyright (c) 2010-2011, Linaro Limited - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of Linaro Limited nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* - Written by Dave Gilbert - - A very simple strchr routine, from benchmarks on A9 it's a bit faster than - the current version in eglibc (2.12.1-0ubuntu14 package) - I don't think doing a word at a time version is worth it since a lot - of strchr cases are very short anyway. - +/*- + * Copyright (c) 2013 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Matt Thomas of 3am Software Foundry. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ #include #include +#ifndef __ARMEB__ +#define BYTE0 0x000000ff +#define BYTE1 0x0000ff00 +#define BYTE2 0x00ff0000 +#define BYTE3 0xff000000 +#define lshi lsl +#define lshis lsls +#else +#define BYTE0 0xff000000 +#define BYTE1 0x00ff0000 +#define BYTE2 0x0000ff00 +#define BYTE3 0x000000ff +#define lshi lsr +#define lshis lsrs +#endif + .syntax unified - .arch armv7-a + .arch armv7-a .thumb - -@ --------------------------------------------------------------------------- - .thumb_func - .p2align 4,,15 -ENTRY(strchr) - @ r0 = start of string - @ r1 = character to match - @ returns NULL for no match, or a pointer to the match - and r1,r1, #255 -1: - ldrb r2,[r0],#1 - cmp r2,r1 - cbz r2,10f - bne 1b - - @ We're here if it matched -5: - subs r0,r0,#1 - bx lr - -10: - @ We're here if we ran off the end - cmp r1, #0 @ Corner case - you're allowed to search for the nil and get a pointer to it - beq 5b @ A bit messy, if it's common we should branch at the start to a special loop - mov r0,#0 +ENTRY(strchr) + and r2, r1, #0xff /* restrict to byte value */ +1: tst r0, #3 /* test for word alignment */ + beq .Lpre_main_loop /* finally word aligned */ + ldrb r3, [r0], #1 /* load a byte */ + cmp r3, r2 /* is it a match? */ + beq 2f /* yes, return current ptr - 1 */ + cmp r3, #0 /* no, was it 0? */ + bne 1b /* no, try next byte */ + movs r0, #0 /* yes, set return value to NULL */ + bx lr /* return */ +2: subs r0, r0, #1 /* back up by one */ + bx lr /* return */ +.Lpre_main_loop: + movw ip, #0xfefe /* magic constant; 254 in each byte */ + movt ip, #0xfefe /* magic constant; 254 in each byte */ + orr r2, r2, r2, lsl #8 /* move to next byte */ + orr r2, r2, r2, lsl #16 /* move to next halfword */ +.Lmain_loop: + ldr r3, [r0], #4 /* load next word */ + /* + * Add 254 to each byte using the UQADD8 (unsigned saturating add 8) + * instruction. For every non-NUL byte, the result for that byte will + * become 255. For NUL, it will be 254. When we complement the + * result, if the result is non-0 then we must have encountered a NUL. + */ + uqadd8 r1, r3, ip /* NUL detection happens here */ + eors r3, r3, r2 /* xor to clear each lane */ + uqadd8 r3, r3, ip /* char detection happens here */ + ands r3, r3, r1 /* merge results */ + mvns r3, r3 /* is the complement non-0? */ + beq .Lmain_loop /* no, then keep going */ + + /* + * We've encountered a NUL or a match but we don't know which happened + * first. + */ + cmp r2, #0 /* searching for NUL? */ + beq .Lfind_match /* yes, find the match */ + mvns r1, r1 /* did we encounter a NUL? */ + beq .Lfind_match /* no, find the match */ + bics r3, r3, r1 /* clear match for the NUL(s) */ + beq .Lnomatch /* any left set? if not, no match */ + lshis r1, r1, #8 /* replicate NUL bit to other bytes */ + itt ne + orrne r1, r1, r1, lshi #8 /* replicate NUL bit to other bytes */ + orrne r1, r1, r1, lshi #8 /* replicate NUL bit to other bytes */ + bics r3, r3, r1 /* clear any match bits after the NUL */ + beq .Lnomatch /* any left set? if not, no match */ +.Lfind_match: +#ifndef __ARMEB__ + rev r3, r3 /* we want this in BE for the CLZ */ +#endif + clz r3, r3 /* count how many leading zeros */ + add r0, r0, r3, lsr #3 /* divide that by 8 and add to count */ + subs r0, r0, #4 /* compensate for the post-inc */ + bx lr +.Lnomatch: + movs r0, #0 bx lr END(strchr) From b2b91d67358fefa47b49ec23954baa708d27efd3 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Mon, 27 Apr 2015 15:36:58 -0500 Subject: [PATCH 63/89] libc: Don't build malloc leak check for userdebug. Change-Id: I067fed514cf427d4b4de8c29586f3452b5af2aec Signed-off-by: W4TCH0UT --- libc/Android.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libc/Android.mk b/libc/Android.mk index bfa8e41f4..095c3c179 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -1124,7 +1124,7 @@ LOCAL_STATIC_LIBRARIES_arm := libc++ LOCAL_ALLOW_UNDEFINED_SYMBOLS := true # Don't install on release build -LOCAL_MODULE_TAGS := eng debug +LOCAL_MODULE_TAGS := eng $(eval $(call patch-up-arch-specific-flags,LOCAL_CFLAGS,libc_common_cflags)) include $(BUILD_SHARED_LIBRARY) @@ -1156,7 +1156,7 @@ LOCAL_SHARED_LIBRARIES := libc libdl LOCAL_SYSTEM_SHARED_LIBRARIES := # Don't install on release build -LOCAL_MODULE_TAGS := eng debug +LOCAL_MODULE_TAGS := eng $(eval $(call patch-up-arch-specific-flags,LOCAL_CFLAGS,libc_common_cflags)) include $(BUILD_SHARED_LIBRARY) From 1c5654044bcbee8d372628467491c04263a1e525 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Mon, 27 Apr 2015 14:31:32 -0500 Subject: [PATCH 64/89] libc: arm: Update libgcc_compat. Change-Id: I2d93bdb1be930c6191cd9551f594dee2194d271d Signed-off-by: W4TCH0UT --- libc/arch-arm/bionic/libgcc_compat.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libc/arch-arm/bionic/libgcc_compat.c b/libc/arch-arm/bionic/libgcc_compat.c index 10faf2d1b..61a6d2476 100644 --- a/libc/arch-arm/bionic/libgcc_compat.c +++ b/libc/arch-arm/bionic/libgcc_compat.c @@ -22,6 +22,7 @@ extern char __aeabi_f2d; extern char __aeabi_f2iz; extern char __aeabi_f2uiz; extern char __aeabi_fadd; +extern char __aeabi_fcmpgt; extern char __aeabi_fcmpun; extern char __aeabi_fdiv; extern char __aeabi_fmul; @@ -101,6 +102,7 @@ void* __bionic_libgcc_compat_symbols[] = { &__aeabi_f2iz, &__aeabi_f2uiz, &__aeabi_fadd, + &__aeabi_fcmpgt, &__aeabi_fcmpun, &__aeabi_fdiv, &__aeabi_fmul, From de53dccd8c1a5255727b433f3788ef6245cb71c6 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Mon, 27 Apr 2015 17:04:57 -0500 Subject: [PATCH 65/89] libc: arm: Add __aeabi_d2uiz to libgcc_compat. Change-Id: Ibd68d339c5e6e435486001874bb6575302d0f98f Signed-off-by: W4TCH0UT --- libc/arch-arm/bionic/libgcc_compat.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libc/arch-arm/bionic/libgcc_compat.c b/libc/arch-arm/bionic/libgcc_compat.c index 61a6d2476..88ed81389 100644 --- a/libc/arch-arm/bionic/libgcc_compat.c +++ b/libc/arch-arm/bionic/libgcc_compat.c @@ -7,6 +7,7 @@ extern char __aeabi_cdcmple; extern char __aeabi_cdrcmple; extern char __aeabi_d2f; extern char __aeabi_d2iz; +extern char __aeabi_d2uiz; extern char __aeabi_dadd; extern char __aeabi_dcmpeq; extern char __aeabi_dcmpge; @@ -87,6 +88,7 @@ void* __bionic_libgcc_compat_symbols[] = { &__aeabi_cdrcmple, &__aeabi_d2f, &__aeabi_d2iz, + &__aeabi_d2uiz, &__aeabi_dadd, &__aeabi_dcmpeq, &__aeabi_dcmpge, From d27246fb52838f875f851b55103d2d9762452092 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Tue, 28 Apr 2015 13:39:41 -0500 Subject: [PATCH 66/89] cortex-a9: Fix reference to __memcpy_base_aligned. With a different memcpy, __memcpy_base_aligned ceased to exist. Instead, point to the name defined by whatever includes memcpy_base.S Change-Id: I242cf49cbada35337ba155d7f170e86a905ff55f Signed-off-by: W4TCH0UT --- libc/arch-arm/cortex-a9/bionic/memcpy_base.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/arch-arm/cortex-a9/bionic/memcpy_base.S b/libc/arch-arm/cortex-a9/bionic/memcpy_base.S index 5e813050a..bb08b94e9 100644 --- a/libc/arch-arm/cortex-a9/bionic/memcpy_base.S +++ b/libc/arch-arm/cortex-a9/bionic/memcpy_base.S @@ -44,7 +44,7 @@ ENTRY_PRIVATE(MEMCPY_BASE) /* check if buffers are aligned. If so, run arm-only version */ eor r3, r0, r1 ands r3, r3, #0x3 - beq __memcpy_base_aligned + beq MEMCPY_BASE_ALIGNED /* Check the upper size limit for Neon unaligned memory access in memcpy */ cmp r2, #224 From 7d41fac71bba0a2a499fed94cf08deae86750805 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Wed, 5 Nov 2014 08:39:54 -0600 Subject: [PATCH 67/89] Add md5 back to libc/bionic for targets that need it. In particular, Galaxy Nexus RIL breaks without this. Signed-off-by: W4TCH0UT Conflicts: libc/bionic/md5.c libc/bionic/md5.h Change-Id: I13ae801f2de1f6350ef54e95c306f3f0af36cf4a Signed-off-by: W4TCH0UT --- libc/Android.mk | 4 ++++ libc/bionic/md5.c | 4 ++++ libc/bionic/md5.h | 7 +++++++ 3 files changed, 15 insertions(+) diff --git a/libc/Android.mk b/libc/Android.mk index 095c3c179..9bd01d970 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -89,6 +89,10 @@ libc_common_src_files += \ bionic/__vsnprintf_chk.cpp \ bionic/__vsprintf_chk.cpp \ +ifeq ($(TARGET_NEEDS_BIONIC_MD5),true) + libc_common_src_files += bionic/md5.c +endif + libc_bionic_src_files := \ bionic/abort.cpp \ bionic/accept.cpp \ diff --git a/libc/bionic/md5.c b/libc/bionic/md5.c index ba4aaed22..82bd576ad 100644 --- a/libc/bionic/md5.c +++ b/libc/bionic/md5.c @@ -231,7 +231,11 @@ MD5_Update (struct md5 *m, const void *v, size_t len) } calc(m, current); #else +<<<<<<< HEAD calc(m, (u_int32_t*)m->save); +======= + calc(m, m->save32); +>>>>>>> b116fdf... Add md5 back to libc/bionic for targets that need it. #endif offset = 0; } diff --git a/libc/bionic/md5.h b/libc/bionic/md5.h index a381994ef..ddffb875b 100644 --- a/libc/bionic/md5.h +++ b/libc/bionic/md5.h @@ -40,7 +40,14 @@ struct md5 { unsigned int sz[2]; u_int32_t counter[4]; +<<<<<<< HEAD unsigned char save[64]; +======= + union { + unsigned char save[64]; + u_int32_t save32[16]; + }; +>>>>>>> b116fdf... Add md5 back to libc/bionic for targets that need it. }; typedef struct md5 MD5_CTX; From 5473db39a80234f02ba396d69e3c2a7035fb1569 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Wed, 15 Apr 2015 23:14:36 -0500 Subject: [PATCH 68/89] libc: Fix strict-aliasing violations. Signed-off-by: W4TCH0UT --- libc/bionic/mbstate.cpp | 4 ++-- libc/bionic/wchar.cpp | 2 +- libc/dns/net/getaddrinfo.c | 2 +- libc/include/netinet/in6.h | 36 ++++++++++++++++++------------------ libc/include/wchar.h | 5 ++++- 5 files changed, 26 insertions(+), 23 deletions(-) diff --git a/libc/bionic/mbstate.cpp b/libc/bionic/mbstate.cpp index cb327d839..35ae76a3c 100644 --- a/libc/bionic/mbstate.cpp +++ b/libc/bionic/mbstate.cpp @@ -47,11 +47,11 @@ __LIBC_HIDDEN__ uint8_t mbstate_get_byte(const mbstate_t* ps, int n) { __LIBC_HIDDEN__ size_t reset_and_return_illegal(int _errno, mbstate_t* ps) { errno = _errno; - *(reinterpret_cast(ps->__seq)) = 0; + ps->__seq32 = 0; return __MB_ERR_ILLEGAL_SEQUENCE; } __LIBC_HIDDEN__ size_t reset_and_return(int _return, mbstate_t* ps) { - *(reinterpret_cast(ps->__seq)) = 0; + ps->__seq32 = 0; return _return; } diff --git a/libc/bionic/wchar.cpp b/libc/bionic/wchar.cpp index e0879b9df..723639cb8 100644 --- a/libc/bionic/wchar.cpp +++ b/libc/bionic/wchar.cpp @@ -54,7 +54,7 @@ // int mbsinit(const mbstate_t* ps) { - return (ps == NULL || (*(reinterpret_cast(ps->__seq)) == 0)); + return (ps == NULL || ps->__seq32 == 0); } size_t mbrtowc(wchar_t* pwc, const char* s, size_t n, mbstate_t* ps) { diff --git a/libc/dns/net/getaddrinfo.c b/libc/dns/net/getaddrinfo.c index 2612d6a4d..ea1b5695f 100644 --- a/libc/dns/net/getaddrinfo.c +++ b/libc/dns/net/getaddrinfo.c @@ -1544,7 +1544,7 @@ _get_scope(const struct sockaddr *addr) /* RFC 4380, section 2.6 */ #define IN6_IS_ADDR_TEREDO(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == ntohl(0x20010000))) + (((a)->s6_addr32[0]) == ntohl(0x20010000)) /* RFC 3056, section 2. */ #define IN6_IS_ADDR_6TO4(a) \ diff --git a/libc/include/netinet/in6.h b/libc/include/netinet/in6.h index 7f3286ae9..f33e41ff5 100644 --- a/libc/include/netinet/in6.h +++ b/libc/include/netinet/in6.h @@ -31,28 +31,28 @@ #include #define IN6_IS_ADDR_UNSPECIFIED(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[4]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[8]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[12]) == 0)) + (((a)->s6_addr32[0] == 0) && \ + ((a)->s6_addr32[1] == 0) && \ + ((a)->s6_addr32[2] == 0) && \ + ((a)->s6_addr32[3] == 0)) #define IN6_IS_ADDR_LOOPBACK(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[4]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[8]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[12]) == ntohl(1))) + (((a)->s6_addr32[0] == 0) && \ + ((a)->s6_addr32[1] == 0) && \ + ((a)->s6_addr32[2] == 0) && \ + ((a)->s6_addr32[3] == ntohl(1))) #define IN6_IS_ADDR_V4COMPAT(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[4]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[8]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[12]) != 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[12]) != ntohl(1))) + (((a)->s6_addr32[0] == 0) && \ + ((a)->s6_addr32[1] == 0) && \ + ((a)->s6_addr32[2] == 0) && \ + ((a)->s6_addr32[3] != 0) && \ + ((a)->s6_addr32[3] != ntohl(1))) -#define IN6_IS_ADDR_V4MAPPED(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[4]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[8]) == ntohl(0x0000ffff))) +#define IN6_IS_ADDR_V4MAPPED(a) \ + (((a)->s6_addr32[0] == 0) && \ + ((a)->s6_addr32[1] == 0) && \ + ((a)->s6_addr32[2] == ntohl(0x0000ffff))) #define IN6_IS_ADDR_LINKLOCAL(a) \ (((a)->s6_addr[0] == 0xfe) && (((a)->s6_addr[1] & 0xc0) == 0x80)) @@ -65,7 +65,7 @@ (((a)->s6_addr[0] & 0xfe) == 0xfc) #define IN6_IS_ADDR_MULTICAST(a) \ - (((__const uint8_t *) (a))[0] == 0xff) + ((a)->s6_addr[0] == 0xff) #define IPV6_ADDR_SCOPE_NODELOCAL 0x01 diff --git a/libc/include/wchar.h b/libc/include/wchar.h index e0e5c82fe..dc328889f 100644 --- a/libc/include/wchar.h +++ b/libc/include/wchar.h @@ -41,7 +41,10 @@ __BEGIN_DECLS typedef __WINT_TYPE__ wint_t; typedef struct { - uint8_t __seq[4]; + union { + uint8_t __seq[4]; + uint32_t __seq32; + }; #ifdef __LP64__ char __reserved[4]; #endif From 7cf23a1d80dbdf740ff763c5912bb1a5f0535782 Mon Sep 17 00:00:00 2001 From: Yingshiuan Pan Date: Wed, 18 Mar 2015 16:08:05 +0800 Subject: [PATCH 69/89] libc: arm: a15: add optimized memcpy implementation This optimization is extracted from cortex-strings and bionic-ized, then applied to Cortex-A15 cpu. This optimization requires NEON for armv7-a or vfpv2 for armv6. NEON is optional in cortex-a9, so I did not change cortex-a9.mk. This new implementation does not have __memcpy_chk(), so __memcpy_chk.cpp is included for arch which uses the new implementation. Change-Id: If3e11f9df4694dee3d3d8a5297817d1bc2a41d6e Signed-off-by: Yingshiuan Pan Signed-off-by: arter97 Signed-off-by: W4TCH0UT Conflicts: libc/arch-arm/cortex-a15/cortex-a15.mk Signed-off-by: W4TCH0UT --- libc/arch-arm/bionic/{memcpy.a9.S => memcpy.neon.S} | 10 ++++------ libc/arch-arm/cortex-a15/cortex-a15.mk | 2 ++ 2 files changed, 6 insertions(+), 6 deletions(-) rename libc/arch-arm/bionic/{memcpy.a9.S => memcpy.neon.S} (98%) diff --git a/libc/arch-arm/bionic/memcpy.a9.S b/libc/arch-arm/bionic/memcpy.neon.S similarity index 98% rename from libc/arch-arm/bionic/memcpy.a9.S rename to libc/arch-arm/bionic/memcpy.neon.S index 259701d67..91c9cc6e0 100644 --- a/libc/arch-arm/bionic/memcpy.a9.S +++ b/libc/arch-arm/bionic/memcpy.neon.S @@ -27,7 +27,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ /* @@ -39,11 +38,9 @@ ARMv6 (ARMv7-a if using Neon) ARM state Unaligned accesses - LDRD/STRD support unaligned word accesses */ -#include #include .syntax unified @@ -250,9 +247,9 @@ ENTRY(memcpy) vmov.f32 s0, s0 #endif - /* SRC and DST have the same mutual 32-bit alignment, but we may + /* SRC and DST have the same mutual 64-bit alignment, but we may still need to pre-copy some bytes to get to natural alignment. - We bring DST into full 64-bit alignment. */ + We bring SRC and DST into full 64-bit alignment. */ lsls tmp2, dst, #29 beq 1f rsbs tmp2, tmp2, #0 @@ -351,7 +348,7 @@ ENTRY(memcpy) .Ltail63aligned: /* Count in tmp2. */ /* Copy up to 7 d-words of data. Similar to Ltail63unaligned, but - we know that the src and dest are 32-bit aligned so we can use + we know that the src and dest are 64-bit aligned so we can use LDRD/STRD to improve efficiency. */ /* TMP2 is now negative, but we don't care about that. The bottom six bits still tell us how many bytes are left to copy. */ @@ -611,4 +608,5 @@ ENTRY(memcpy) ldr tmp2, [sp], #FRAME_SIZE bne .Ltail63unaligned bx lr + END(memcpy) diff --git a/libc/arch-arm/cortex-a15/cortex-a15.mk b/libc/arch-arm/cortex-a15/cortex-a15.mk index b4279f802..d82c49cb3 100644 --- a/libc/arch-arm/cortex-a15/cortex-a15.mk +++ b/libc/arch-arm/cortex-a15/cortex-a15.mk @@ -1,5 +1,7 @@ libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/memchr.S \ + bionic/__memcpy_chk.cpp \ + arch-arm/bionic/memcpy.neon.S \ arch-arm/cortex-a15/bionic/memset.S \ arch-arm/cortex-a15/bionic/strcat.S \ arch-arm/cortex-a15/bionic/__strcat_chk.S \ From 2549ca1d58778d4dc3b30efefc9013a9ea022fa6 Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Fri, 12 Sep 2014 14:00:02 -0700 Subject: [PATCH 70/89] Sync libm with upstream. Change-Id: I8ac8ee52122ee19a2e423c3211092023cb4896eb --- libm/Android.mk | 1 + .../lib/msun/ld128/e_rem_pio2l.h | 18 +- .../upstream-freebsd/lib/msun/ld128/invtrig.c | 4 +- libm/upstream-freebsd/lib/msun/ld128/k_cosl.c | 2 +- libm/upstream-freebsd/lib/msun/ld128/k_expl.h | 328 +++++++++++++++++ libm/upstream-freebsd/lib/msun/ld128/k_sinl.c | 2 +- libm/upstream-freebsd/lib/msun/ld128/s_erfl.c | 329 ++++++++++++++++++ .../upstream-freebsd/lib/msun/ld128/s_exp2l.c | 2 +- libm/upstream-freebsd/lib/msun/ld128/s_expl.c | 298 ++++------------ libm/upstream-freebsd/lib/msun/src/e_cosh.c | 6 + .../lib/msun/src/e_lgamma_r.c | 64 ++-- .../lib/msun/src/e_lgammaf_r.c | 57 ++- libm/upstream-freebsd/lib/msun/src/e_pow.c | 8 +- libm/upstream-freebsd/lib/msun/src/e_sinh.c | 6 + .../upstream-freebsd/lib/msun/src/imprecise.c | 2 - libm/upstream-freebsd/lib/msun/src/s_erf.c | 64 ++-- libm/upstream-freebsd/lib/msun/src/s_erff.c | 127 ++++--- libm/upstream-freebsd/lib/msun/src/s_rintl.c | 1 - libm/upstream-freebsd/lib/msun/src/s_tanh.c | 9 +- libm/upstream-freebsd/lib/msun/src/s_tanhf.c | 4 +- 20 files changed, 919 insertions(+), 413 deletions(-) create mode 100644 libm/upstream-freebsd/lib/msun/ld128/k_expl.h create mode 100644 libm/upstream-freebsd/lib/msun/ld128/s_erfl.c diff --git a/libm/Android.mk b/libm/Android.mk index 30c2bc7fc..379aea70a 100644 --- a/libm/Android.mk +++ b/libm/Android.mk @@ -220,6 +220,7 @@ libm_ld_src_files += \ upstream-freebsd/lib/msun/ld128/k_cosl.c \ upstream-freebsd/lib/msun/ld128/k_sinl.c \ upstream-freebsd/lib/msun/ld128/k_tanl.c \ + upstream-freebsd/lib/msun/ld128/s_erfl.c \ upstream-freebsd/lib/msun/ld128/s_exp2l.c \ upstream-freebsd/lib/msun/ld128/s_expl.c \ upstream-freebsd/lib/msun/ld128/s_logl.c \ diff --git a/libm/upstream-freebsd/lib/msun/ld128/e_rem_pio2l.h b/libm/upstream-freebsd/lib/msun/ld128/e_rem_pio2l.h index 078d0c3d4..5d78c4de0 100644 --- a/libm/upstream-freebsd/lib/msun/ld128/e_rem_pio2l.h +++ b/libm/upstream-freebsd/lib/msun/ld128/e_rem_pio2l.h @@ -6,7 +6,7 @@ * * Developed at SunSoft, a Sun Microsystems, Inc. business. * Permission to use, copy, modify, and distribute this - * software is freely granted, provided that this notice + * software is freely granted, provided that this notice * is preserved. * ==================================================== * @@ -17,8 +17,8 @@ __FBSDID("$FreeBSD$"); /* ld128 version of __ieee754_rem_pio2l(x,y) - * - * return the remainder of x rem pi/2 in y[0]+y[1] + * + * return the remainder of x rem pi/2 in y[0]+y[1] * use __kernel_rem_pio2() */ @@ -88,24 +88,24 @@ __ieee754_rem_pio2l(long double x, long double *y) union IEEEl2bits u2; int ex1; j = ex; - y[0] = r-w; + y[0] = r-w; u2.e = y[0]; ex1 = u2.xbits.expsign & 0x7fff; i = j-ex1; if(i>51) { /* 2nd iteration needed, good to 248 */ t = r; - w = fn*pio2_2; + w = fn*pio2_2; r = t-w; - w = fn*pio2_2t-((t-r)-w); + w = fn*pio2_2t-((t-r)-w); y[0] = r-w; u2.e = y[0]; ex1 = u2.xbits.expsign & 0x7fff; i = j-ex1; if(i>119) { /* 3rd iteration need, 316 bits acc */ t = r; /* will cover all possible cases */ - w = fn*pio2_3; + w = fn*pio2_3; r = t-w; - w = fn*pio2_3t-((t-r)-w); + w = fn*pio2_3t-((t-r)-w); y[0] = r-w; } } @@ -113,7 +113,7 @@ __ieee754_rem_pio2l(long double x, long double *y) y[1] = (r-y[0])-w; return n; } - /* + /* * all other (large) arguments */ if(ex==0x7fff) { /* x is inf or NaN */ diff --git a/libm/upstream-freebsd/lib/msun/ld128/invtrig.c b/libm/upstream-freebsd/lib/msun/ld128/invtrig.c index df67d16f5..4ceca8a45 100644 --- a/libm/upstream-freebsd/lib/msun/ld128/invtrig.c +++ b/libm/upstream-freebsd/lib/msun/ld128/invtrig.c @@ -58,8 +58,8 @@ qS9 = -1.99407384882605586705979504567947007e-04L; */ const long double atanhi[] = { 4.63647609000806116214256231461214397e-01L, - 7.85398163397448309615660845819875699e-01L, - 9.82793723247329067985710611014666038e-01L, + 7.85398163397448309615660845819875699e-01L, + 9.82793723247329067985710611014666038e-01L, 1.57079632679489661923132169163975140e+00L, }; diff --git a/libm/upstream-freebsd/lib/msun/ld128/k_cosl.c b/libm/upstream-freebsd/lib/msun/ld128/k_cosl.c index 5f4aa378d..fe57773a1 100644 --- a/libm/upstream-freebsd/lib/msun/ld128/k_cosl.c +++ b/libm/upstream-freebsd/lib/msun/ld128/k_cosl.c @@ -6,7 +6,7 @@ * * Developed at SunSoft, a Sun Microsystems, Inc. business. * Permission to use, copy, modify, and distribute this - * software is freely granted, provided that this notice + * software is freely granted, provided that this notice * is preserved. * ==================================================== */ diff --git a/libm/upstream-freebsd/lib/msun/ld128/k_expl.h b/libm/upstream-freebsd/lib/msun/ld128/k_expl.h new file mode 100644 index 000000000..a5668fd47 --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/ld128/k_expl.h @@ -0,0 +1,328 @@ +/* from: FreeBSD: head/lib/msun/ld128/s_expl.c 251345 2013-06-03 20:09:22Z kargl */ + +/*- + * Copyright (c) 2009-2013 Steven G. Kargl + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice unmodified, this list of conditions, and the following + * disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Optimized by Bruce D. Evans. + */ + +#include +__FBSDID("$FreeBSD$"); + +/* + * ld128 version of k_expl.h. See ../ld80/s_expl.c for most comments. + * + * See ../src/e_exp.c and ../src/k_exp.h for precision-independent comments + * about the secondary kernels. + */ + +#define INTERVALS 128 +#define LOG2_INTERVALS 7 +#define BIAS (LDBL_MAX_EXP - 1) + +static const double +/* + * ln2/INTERVALS = L1+L2 (hi+lo decomposition for multiplication). L1 must + * have at least 22 (= log2(|LDBL_MIN_EXP-extras|) + log2(INTERVALS)) lowest + * bits zero so that multiplication of it by n is exact. + */ +INV_L = 1.8466496523378731e+2, /* 0x171547652b82fe.0p-45 */ +L2 = -1.0253670638894731e-29; /* -0x1.9ff0342542fc3p-97 */ +static const long double +/* 0x1.62e42fefa39ef35793c768000000p-8 */ +L1 = 5.41521234812457272982212595914567508e-3L; + +/* + * XXX values in hex in comments have been lost (or were never present) + * from here. + */ +static const long double +/* + * Domain [-0.002708, 0.002708], range ~[-2.4021e-38, 2.4234e-38]: + * |exp(x) - p(x)| < 2**-124.9 + * (0.002708 is ln2/(2*INTERVALS) rounded up a little). + * + * XXX the coeffs aren't very carefully rounded, and I get 3.6 more bits. + */ +A2 = 0.5, +A3 = 1.66666666666666666666666666651085500e-1L, +A4 = 4.16666666666666666666666666425885320e-2L, +A5 = 8.33333333333333333334522877160175842e-3L, +A6 = 1.38888888888888888889971139751596836e-3L; + +static const double +A7 = 1.9841269841269470e-4, /* 0x1.a01a01a019f91p-13 */ +A8 = 2.4801587301585286e-5, /* 0x1.71de3ec75a967p-19 */ +A9 = 2.7557324277411235e-6, /* 0x1.71de3ec75a967p-19 */ +A10 = 2.7557333722375069e-7; /* 0x1.27e505ab56259p-22 */ + +static const struct { + /* + * hi must be rounded to at most 106 bits so that multiplication + * by r1 in expm1l() is exact, but it is rounded to 88 bits due to + * historical accidents. + * + * XXX it is wasteful to use long double for both hi and lo. ld128 + * exp2l() uses only float for lo (in a very differently organized + * table; ld80 exp2l() is different again. It uses 2 doubles in a + * table organized like this one. 1 double and 1 float would + * suffice). There are different packing/locality/alignment/caching + * problems with these methods. + * + * XXX C's bad %a format makes the bits unreadable. They happen + * to all line up for the hi values 1 before the point and 88 + * in 22 nybbles, but for the low values the nybbles are shifted + * randomly. + */ + long double hi; + long double lo; +} tbl[INTERVALS] = { + 0x1p0L, 0x0p0L, + 0x1.0163da9fb33356d84a66aep0L, 0x3.36dcdfa4003ec04c360be2404078p-92L, + 0x1.02c9a3e778060ee6f7cacap0L, 0x4.f7a29bde93d70a2cabc5cb89ba10p-92L, + 0x1.04315e86e7f84bd738f9a2p0L, 0xd.a47e6ed040bb4bfc05af6455e9b8p-96L, + 0x1.059b0d31585743ae7c548ep0L, 0xb.68ca417fe53e3495f7df4baf84a0p-92L, + 0x1.0706b29ddf6ddc6dc403a8p0L, 0x1.d87b27ed07cb8b092ac75e311753p-88L, + 0x1.0874518759bc808c35f25cp0L, 0x1.9427fa2b041b2d6829d8993a0d01p-88L, + 0x1.09e3ecac6f3834521e060cp0L, 0x5.84d6b74ba2e023da730e7fccb758p-92L, + 0x1.0b5586cf9890f6298b92b6p0L, 0x1.1842a98364291408b3ceb0a2a2bbp-88L, + 0x1.0cc922b7247f7407b705b8p0L, 0x9.3dc5e8aac564e6fe2ef1d431fd98p-92L, + 0x1.0e3ec32d3d1a2020742e4ep0L, 0x1.8af6a552ac4b358b1129e9f966a4p-88L, + 0x1.0fb66affed31af232091dcp0L, 0x1.8a1426514e0b627bda694a400a27p-88L, + 0x1.11301d0125b50a4ebbf1aep0L, 0xd.9318ceac5cc47ab166ee57427178p-92L, + 0x1.12abdc06c31cbfb92bad32p0L, 0x4.d68e2f7270bdf7cedf94eb1cb818p-92L, + 0x1.1429aaea92ddfb34101942p0L, 0x1.b2586d01844b389bea7aedd221d4p-88L, + 0x1.15a98c8a58e512480d573cp0L, 0x1.d5613bf92a2b618ee31b376c2689p-88L, + 0x1.172b83c7d517adcdf7c8c4p0L, 0x1.0eb14a792035509ff7d758693f24p-88L, + 0x1.18af9388c8de9bbbf70b9ap0L, 0x3.c2505c97c0102e5f1211941d2840p-92L, + 0x1.1a35beb6fcb753cb698f68p0L, 0x1.2d1c835a6c30724d5cfae31b84e5p-88L, + 0x1.1bbe084045cd39ab1e72b4p0L, 0x4.27e35f9acb57e473915519a1b448p-92L, + 0x1.1d4873168b9aa7805b8028p0L, 0x9.90f07a98b42206e46166cf051d70p-92L, + 0x1.1ed5022fcd91cb8819ff60p0L, 0x1.121d1e504d36c47474c9b7de6067p-88L, + 0x1.2063b88628cd63b8eeb028p0L, 0x1.50929d0fc487d21c2b84004264dep-88L, + 0x1.21f49917ddc962552fd292p0L, 0x9.4bdb4b61ea62477caa1dce823ba0p-92L, + 0x1.2387a6e75623866c1fadb0p0L, 0x1.c15cb593b0328566902df69e4de2p-88L, + 0x1.251ce4fb2a63f3582ab7dep0L, 0x9.e94811a9c8afdcf796934bc652d0p-92L, + 0x1.26b4565e27cdd257a67328p0L, 0x1.d3b249dce4e9186ddd5ff44e6b08p-92L, + 0x1.284dfe1f5638096cf15cf0p0L, 0x3.ca0967fdaa2e52d7c8106f2e262cp-92L, + 0x1.29e9df51fdee12c25d15f4p0L, 0x1.a24aa3bca890ac08d203fed80a07p-88L, + 0x1.2b87fd0dad98ffddea4652p0L, 0x1.8fcab88442fdc3cb6de4519165edp-88L, + 0x1.2d285a6e4030b40091d536p0L, 0xd.075384589c1cd1b3e4018a6b1348p-92L, + 0x1.2ecafa93e2f5611ca0f45cp0L, 0x1.523833af611bdcda253c554cf278p-88L, + 0x1.306fe0a31b7152de8d5a46p0L, 0x3.05c85edecbc27343629f502f1af2p-92L, + 0x1.32170fc4cd8313539cf1c2p0L, 0x1.008f86dde3220ae17a005b6412bep-88L, + 0x1.33c08b26416ff4c9c8610cp0L, 0x1.96696bf95d1593039539d94d662bp-88L, + 0x1.356c55f929ff0c94623476p0L, 0x3.73af38d6d8d6f9506c9bbc93cbc0p-92L, + 0x1.371a7373aa9caa7145502ep0L, 0x1.4547987e3e12516bf9c699be432fp-88L, + 0x1.38cae6d05d86585a9cb0d8p0L, 0x1.bed0c853bd30a02790931eb2e8f0p-88L, + 0x1.3a7db34e59ff6ea1bc9298p0L, 0x1.e0a1d336163fe2f852ceeb134067p-88L, + 0x1.3c32dc313a8e484001f228p0L, 0xb.58f3775e06ab66353001fae9fca0p-92L, + 0x1.3dea64c12342235b41223ep0L, 0x1.3d773fba2cb82b8244267c54443fp-92L, + 0x1.3fa4504ac801ba0bf701aap0L, 0x4.1832fb8c1c8dbdff2c49909e6c60p-92L, + 0x1.4160a21f72e29f84325b8ep0L, 0x1.3db61fb352f0540e6ba05634413ep-88L, + 0x1.431f5d950a896dc7044394p0L, 0x1.0ccec81e24b0caff7581ef4127f7p-92L, + 0x1.44e086061892d03136f408p0L, 0x1.df019fbd4f3b48709b78591d5cb5p-88L, + 0x1.46a41ed1d005772512f458p0L, 0x1.229d97df404ff21f39c1b594d3a8p-88L, + 0x1.486a2b5c13cd013c1a3b68p0L, 0x1.062f03c3dd75ce8757f780e6ec99p-88L, + 0x1.4a32af0d7d3de672d8bcf4p0L, 0x6.f9586461db1d878b1d148bd3ccb8p-92L, + 0x1.4bfdad5362a271d4397afep0L, 0xc.42e20e0363ba2e159c579f82e4b0p-92L, + 0x1.4dcb299fddd0d63b36ef1ap0L, 0x9.e0cc484b25a5566d0bd5f58ad238p-92L, + 0x1.4f9b2769d2ca6ad33d8b68p0L, 0x1.aa073ee55e028497a329a7333dbap-88L, + 0x1.516daa2cf6641c112f52c8p0L, 0x4.d822190e718226177d7608d20038p-92L, + 0x1.5342b569d4f81df0a83c48p0L, 0x1.d86a63f4e672a3e429805b049465p-88L, + 0x1.551a4ca5d920ec52ec6202p0L, 0x4.34ca672645dc6c124d6619a87574p-92L, + 0x1.56f4736b527da66ecb0046p0L, 0x1.64eb3c00f2f5ab3d801d7cc7272dp-88L, + 0x1.58d12d497c7fd252bc2b72p0L, 0x1.43bcf2ec936a970d9cc266f0072fp-88L, + 0x1.5ab07dd48542958c930150p0L, 0x1.91eb345d88d7c81280e069fbdb63p-88L, + 0x1.5c9268a5946b701c4b1b80p0L, 0x1.6986a203d84e6a4a92f179e71889p-88L, + 0x1.5e76f15ad21486e9be4c20p0L, 0x3.99766a06548a05829e853bdb2b52p-92L, + 0x1.605e1b976dc08b076f592ap0L, 0x4.86e3b34ead1b4769df867b9c89ccp-92L, + 0x1.6247eb03a5584b1f0fa06ep0L, 0x1.d2da42bb1ceaf9f732275b8aef30p-88L, + 0x1.6434634ccc31fc76f8714cp0L, 0x4.ed9a4e41000307103a18cf7a6e08p-92L, + 0x1.66238825522249127d9e28p0L, 0x1.b8f314a337f4dc0a3adf1787ff74p-88L, + 0x1.68155d44ca973081c57226p0L, 0x1.b9f32706bfe4e627d809a85dcc66p-88L, + 0x1.6a09e667f3bcc908b2fb12p0L, 0x1.66ea957d3e3adec17512775099dap-88L, + 0x1.6c012750bdabeed76a9980p0L, 0xf.4f33fdeb8b0ecd831106f57b3d00p-96L, + 0x1.6dfb23c651a2ef220e2cbep0L, 0x1.bbaa834b3f11577ceefbe6c1c411p-92L, + 0x1.6ff7df9519483cf87e1b4ep0L, 0x1.3e213bff9b702d5aa477c12523cep-88L, + 0x1.71f75e8ec5f73dd2370f2ep0L, 0xf.0acd6cb434b562d9e8a20adda648p-92L, + 0x1.73f9a48a58173bd5c9a4e6p0L, 0x8.ab1182ae217f3a7681759553e840p-92L, + 0x1.75feb564267c8bf6e9aa32p0L, 0x1.a48b27071805e61a17b954a2dad8p-88L, + 0x1.780694fde5d3f619ae0280p0L, 0x8.58b2bb2bdcf86cd08e35fb04c0f0p-92L, + 0x1.7a11473eb0186d7d51023ep0L, 0x1.6cda1f5ef42b66977960531e821bp-88L, + 0x1.7c1ed0130c1327c4933444p0L, 0x1.937562b2dc933d44fc828efd4c9cp-88L, + 0x1.7e2f336cf4e62105d02ba0p0L, 0x1.5797e170a1427f8fcdf5f3906108p-88L, + 0x1.80427543e1a11b60de6764p0L, 0x9.a354ea706b8e4d8b718a672bf7c8p-92L, + 0x1.82589994cce128acf88afap0L, 0xb.34a010f6ad65cbbac0f532d39be0p-92L, + 0x1.8471a4623c7acce52f6b96p0L, 0x1.c64095370f51f48817914dd78665p-88L, + 0x1.868d99b4492ec80e41d90ap0L, 0xc.251707484d73f136fb5779656b70p-92L, + 0x1.88ac7d98a669966530bcdep0L, 0x1.2d4e9d61283ef385de170ab20f96p-88L, + 0x1.8ace5422aa0db5ba7c55a0p0L, 0x1.92c9bb3e6ed61f2733304a346d8fp-88L, + 0x1.8cf3216b5448bef2aa1cd0p0L, 0x1.61c55d84a9848f8c453b3ca8c946p-88L, + 0x1.8f1ae991577362b982745cp0L, 0x7.2ed804efc9b4ae1458ae946099d4p-92L, + 0x1.9145b0b91ffc588a61b468p0L, 0x1.f6b70e01c2a90229a4c4309ea719p-88L, + 0x1.93737b0cdc5e4f4501c3f2p0L, 0x5.40a22d2fc4af581b63e8326efe9cp-92L, + 0x1.95a44cbc8520ee9b483694p0L, 0x1.a0fc6f7c7d61b2b3a22a0eab2cadp-88L, + 0x1.97d829fde4e4f8b9e920f8p0L, 0x1.1e8bd7edb9d7144b6f6818084cc7p-88L, + 0x1.9a0f170ca07b9ba3109b8cp0L, 0x4.6737beb19e1eada6825d3c557428p-92L, + 0x1.9c49182a3f0901c7c46b06p0L, 0x1.1f2be58ddade50c217186c90b457p-88L, + 0x1.9e86319e323231824ca78ep0L, 0x6.4c6e010f92c082bbadfaf605cfd4p-92L, + 0x1.a0c667b5de564b29ada8b8p0L, 0xc.ab349aa0422a8da7d4512edac548p-92L, + 0x1.a309bec4a2d3358c171f76p0L, 0x1.0daad547fa22c26d168ea762d854p-88L, + 0x1.a5503b23e255c8b424491cp0L, 0xa.f87bc8050a405381703ef7caff50p-92L, + 0x1.a799e1330b3586f2dfb2b0p0L, 0x1.58f1a98796ce8908ae852236ca94p-88L, + 0x1.a9e6b5579fdbf43eb243bcp0L, 0x1.ff4c4c58b571cf465caf07b4b9f5p-88L, + 0x1.ac36bbfd3f379c0db966a2p0L, 0x1.1265fc73e480712d20f8597a8e7bp-88L, + 0x1.ae89f995ad3ad5e8734d16p0L, 0x1.73205a7fbc3ae675ea440b162d6cp-88L, + 0x1.b0e07298db66590842acdep0L, 0x1.c6f6ca0e5dcae2aafffa7a0554cbp-88L, + 0x1.b33a2b84f15faf6bfd0e7ap0L, 0x1.d947c2575781dbb49b1237c87b6ep-88L, + 0x1.b59728de559398e3881110p0L, 0x1.64873c7171fefc410416be0a6525p-88L, + 0x1.b7f76f2fb5e46eaa7b081ap0L, 0xb.53c5354c8903c356e4b625aacc28p-92L, + 0x1.ba5b030a10649840cb3c6ap0L, 0xf.5b47f297203757e1cc6eadc8bad0p-92L, + 0x1.bcc1e904bc1d2247ba0f44p0L, 0x1.b3d08cd0b20287092bd59be4ad98p-88L, + 0x1.bf2c25bd71e088408d7024p0L, 0x1.18e3449fa073b356766dfb568ff4p-88L, + 0x1.c199bdd85529c2220cb12ap0L, 0x9.1ba6679444964a36661240043970p-96L, + 0x1.c40ab5fffd07a6d14df820p0L, 0xf.1828a5366fd387a7bdd54cdf7300p-92L, + 0x1.c67f12e57d14b4a2137fd2p0L, 0xf.2b301dd9e6b151a6d1f9d5d5f520p-96L, + 0x1.c8f6d9406e7b511acbc488p0L, 0x5.c442ddb55820171f319d9e5076a8p-96L, + 0x1.cb720dcef90691503cbd1ep0L, 0x9.49db761d9559ac0cb6dd3ed599e0p-92L, + 0x1.cdf0b555dc3f9c44f8958ep0L, 0x1.ac51be515f8c58bdfb6f5740a3a4p-88L, + 0x1.d072d4a07897b8d0f22f20p0L, 0x1.a158e18fbbfc625f09f4cca40874p-88L, + 0x1.d2f87080d89f18ade12398p0L, 0x9.ea2025b4c56553f5cdee4c924728p-92L, + 0x1.d5818dcfba48725da05aeap0L, 0x1.66e0dca9f589f559c0876ff23830p-88L, + 0x1.d80e316c98397bb84f9d04p0L, 0x8.805f84bec614de269900ddf98d28p-92L, + 0x1.da9e603db3285708c01a5ap0L, 0x1.6d4c97f6246f0ec614ec95c99392p-88L, + 0x1.dd321f301b4604b695de3cp0L, 0x6.30a393215299e30d4fb73503c348p-96L, + 0x1.dfc97337b9b5eb968cac38p0L, 0x1.ed291b7225a944efd5bb5524b927p-88L, + 0x1.e264614f5a128a12761fa0p0L, 0x1.7ada6467e77f73bf65e04c95e29dp-88L, + 0x1.e502ee78b3ff6273d13014p0L, 0x1.3991e8f49659e1693be17ae1d2f9p-88L, + 0x1.e7a51fbc74c834b548b282p0L, 0x1.23786758a84f4956354634a416cep-88L, + 0x1.ea4afa2a490d9858f73a18p0L, 0xf.5db301f86dea20610ceee13eb7b8p-92L, + 0x1.ecf482d8e67f08db0312fap0L, 0x1.949cef462010bb4bc4ce72a900dfp-88L, + 0x1.efa1bee615a27771fd21a8p0L, 0x1.2dac1f6dd5d229ff68e46f27e3dfp-88L, + 0x1.f252b376bba974e8696fc2p0L, 0x1.6390d4c6ad5476b5162f40e1d9a9p-88L, + 0x1.f50765b6e4540674f84b76p0L, 0x2.862baff99000dfc4352ba29b8908p-92L, + 0x1.f7bfdad9cbe138913b4bfep0L, 0x7.2bd95c5ce7280fa4d2344a3f5618p-92L, + 0x1.fa7c1819e90d82e90a7e74p0L, 0xb.263c1dc060c36f7650b4c0f233a8p-92L, + 0x1.fd3c22b8f71f10975ba4b2p0L, 0x1.2bcf3a5e12d269d8ad7c1a4a8875p-88L +}; + +/* + * Kernel for expl(x). x must be finite and not tiny or huge. + * "tiny" is anything that would make us underflow (|A6*x^6| < ~LDBL_MIN). + * "huge" is anything that would make fn*L1 inexact (|x| > ~2**17*ln2). + */ +static inline void +__k_expl(long double x, long double *hip, long double *lop, int *kp) +{ + long double q, r, r1, t; + double dr, fn, r2; + int n, n2; + + /* Reduce x to (k*ln2 + endpoint[n2] + r1 + r2). */ + /* Use a specialized rint() to get fn. Assume round-to-nearest. */ + /* XXX assume no extra precision for the additions, as for trig fns. */ + /* XXX this set of comments is now quadruplicated. */ + /* XXX but see ../src/e_exp.c for a fix using double_t. */ + fn = (double)x * INV_L + 0x1.8p52 - 0x1.8p52; +#if defined(HAVE_EFFICIENT_IRINT) + n = irint(fn); +#else + n = (int)fn; +#endif + n2 = (unsigned)n % INTERVALS; + /* Depend on the sign bit being propagated: */ + *kp = n >> LOG2_INTERVALS; + r1 = x - fn * L1; + r2 = fn * -L2; + r = r1 + r2; + + /* Evaluate expl(endpoint[n2] + r1 + r2) = tbl[n2] * expl(r1 + r2). */ + dr = r; + q = r2 + r * r * (A2 + r * (A3 + r * (A4 + r * (A5 + r * (A6 + + dr * (A7 + dr * (A8 + dr * (A9 + dr * A10)))))))); + t = tbl[n2].lo + tbl[n2].hi; + *hip = tbl[n2].hi; + *lop = tbl[n2].lo + t * (q + r1); +} + +/* + * XXX: the rest of the functions are identical for ld80 and ld128. + * However, we should use scalbnl() for ld128, since long double + * multiplication is very slow on the only supported ld128 arch (sparc64). + */ + +static inline void +k_hexpl(long double x, long double *hip, long double *lop) +{ + float twopkm1; + int k; + + __k_expl(x, hip, lop, &k); + SET_FLOAT_WORD(twopkm1, 0x3f800000 + ((k - 1) << 23)); + *hip *= twopkm1; + *lop *= twopkm1; +} + +static inline long double +hexpl(long double x) +{ + long double hi, lo, twopkm2; + int k; + + twopkm2 = 1; + __k_expl(x, &hi, &lo, &k); + SET_LDBL_EXPSIGN(twopkm2, BIAS + k - 2); + return (lo + hi) * 2 * twopkm2; +} + +#ifdef _COMPLEX_H +/* + * See ../src/k_exp.c for details. + */ +static inline long double complex +__ldexp_cexpl(long double complex z, int expt) +{ + long double exp_x, hi, lo; + long double x, y, scale1, scale2; + int half_expt, k; + + x = creall(z); + y = cimagl(z); + __k_expl(x, &hi, &lo, &k); + + exp_x = (lo + hi) * 0x1p16382; + expt += k - 16382; + + scale1 = 1; + half_expt = expt / 2; + SET_LDBL_EXPSIGN(scale1, BIAS + half_expt); + scale2 = 1; + SET_LDBL_EXPSIGN(scale1, BIAS + expt - half_expt); + + return (cpackl(cos(y) * exp_x * scale1 * scale2, + sinl(y) * exp_x * scale1 * scale2)); +} +#endif /* _COMPLEX_H */ diff --git a/libm/upstream-freebsd/lib/msun/ld128/k_sinl.c b/libm/upstream-freebsd/lib/msun/ld128/k_sinl.c index bd415c053..09472d65f 100644 --- a/libm/upstream-freebsd/lib/msun/ld128/k_sinl.c +++ b/libm/upstream-freebsd/lib/msun/ld128/k_sinl.c @@ -6,7 +6,7 @@ * * Developed at SunSoft, a Sun Microsystems, Inc. business. * Permission to use, copy, modify, and distribute this - * software is freely granted, provided that this notice + * software is freely granted, provided that this notice * is preserved. * ==================================================== */ diff --git a/libm/upstream-freebsd/lib/msun/ld128/s_erfl.c b/libm/upstream-freebsd/lib/msun/ld128/s_erfl.c new file mode 100644 index 000000000..e29c9691f --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/ld128/s_erfl.c @@ -0,0 +1,329 @@ +/* @(#)s_erf.c 5.1 93/09/24 */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +#include +__FBSDID("$FreeBSD$"); + +/* + * See s_erf.c for complete comments. + * + * Converted to long double by Steven G. Kargl. + */ +#include + +#include "fpmath.h" +#include "math.h" +#include "math_private.h" + +/* XXX Prevent compilers from erroneously constant folding these: */ +static const volatile long double tiny = 0x1p-10000L; + +static const double +half= 0.5, +one = 1, +two = 2; +/* + * In the domain [0, 2**-40], only the first term in the power series + * expansion of erf(x) is used. The magnitude of the first neglected + * terms is less than 2**-120. + */ +static const long double +efx = 1.28379167095512573896158903121545167e-01L, /* 0xecbff6a7, 0x481dd788, 0xb64d21a8, 0xeb06fc3f */ +efx8 = 1.02703333676410059116927122497236133e+00L, /* 0xecbff6a7, 0x481dd788, 0xb64d21a8, 0xeb06ff3f */ +/* + * Domain [0, 0.84375], range ~[-1.919e-38, 1.919e-38]: + * |(erf(x) - x)/x - pp(x)/qq(x)| < 2**-125.29 + */ +pp0 = 1.28379167095512573896158903121545167e-01L, /* 0x3ffc06eb, 0xa8214db6, 0x88d71d48, 0xa7f6bfec */ +pp1 = -3.14931554396568573802046931159683404e-01L, /* 0xbffd427d, 0x6ada7263, 0x547eb096, 0x95f37463 */ +pp2 = -5.27514920282183487103576956956725309e-02L, /* 0xbffab023, 0xe5a271e3, 0xb0e79b01, 0x2f7ac962 */ +pp3 = -1.13202828509005281355609495523452713e-02L, /* 0xbff872f1, 0x6a5023a1, 0xe08b3884, 0x326af20f */ +pp4 = -9.18626155872522453865998391206048506e-04L, /* 0xbff4e19f, 0xea5fb024, 0x43247a37, 0xe430b06c */ +pp5 = -7.87518862406176274922506447157284230e-05L, /* 0xbff14a4f, 0x31a85fe0, 0x7fff2204, 0x09c49b37 */ +pp6 = -3.42357944472240436548115331090560881e-06L, /* 0xbfeccb81, 0x4b43c336, 0xcd2eb6c2, 0x903f2d87 */ +pp7 = -1.37317432573890412634717890726745428e-07L, /* 0xbfe826e3, 0x0e915eb6, 0x42aee414, 0xf7e36805 */ +pp8 = -2.71115170113861755855049008732113726e-09L, /* 0xbfe2749e, 0x2b94fd00, 0xecb4d166, 0x0efb91f8 */ +pp9 = -3.37925756196555959454018189718117864e-11L, /* 0xbfdc293e, 0x1d9060cb, 0xd043204a, 0x314cd7f0 */ +qq1 = 4.76672625471551170489978555182449450e-01L, /* 0x3ffde81c, 0xde6531f0, 0x76803bee, 0x526e29e9 */ +qq2 = 1.06713144672281502058807525850732240e-01L, /* 0x3ffbb518, 0xd7a6bb74, 0xcd9bdd33, 0x7601eee5 */ +qq3 = 1.47747613127513761102189201923147490e-02L, /* 0x3ff8e423, 0xae527e18, 0xf12cb447, 0x723b4749 */ +qq4 = 1.39939377672028671891148770908874816e-03L, /* 0x3ff56ed7, 0xba055d84, 0xc21b45c4, 0x388d1812 */ +qq5 = 9.44302939359455241271983309378738276e-05L, /* 0x3ff18c11, 0xc18c99a4, 0x86d0fe09, 0x46387b4c */ +qq6 = 4.56199342312522842161301671745365650e-06L, /* 0x3fed3226, 0x73421d05, 0x08875300, 0x32fa1432 */ +qq7 = 1.53019260483764773845294600092361197e-07L, /* 0x3fe8489b, 0x3a63f627, 0x2b9ad2ce, 0x26516e57 */ +qq8 = 3.25542691121324805094777901250005508e-09L, /* 0x3fe2bf6c, 0x26d93a29, 0x9142be7c, 0x9f1dd043 */ +qq9 = 3.37405581964478060434410167262684979e-11L; /* 0x3fdc28c8, 0xfb8fa1be, 0x10e57eec, 0xaa19e49f */ + +static const long double +erx = 8.42700792949714894142232424201210961e-01L, /* 0x3ffeaf76, 0x7a741088, 0xb0000000, 0x00000000 */ +/* + * Domain [0.84375, 1.25], range ~[-2.521e-36, 2.523e-36]: + * |(erf(x) - erx) - pa(x)/qa(x)| < 2**-120.15 + */ +pa0 = -2.48010117891186017024438233323795897e-17L, /* 0xbfc7c97f, 0x77812279, 0x6c877f22, 0xef4bfb2e */ +pa1 = 4.15107497420594680894327969504526489e-01L, /* 0x3ffda911, 0xf096fbc2, 0x55662005, 0x2337fa64 */ +pa2 = -3.94180628087084846724448515851892609e-02L, /* 0xbffa42e9, 0xab54528c, 0xad529da1, 0x6efc2af3 */ +pa3 = 4.48897599625192107295954790681677462e-02L, /* 0x3ffa6fbc, 0xa65edba1, 0x0e4cbcea, 0x73ef9a31 */ +pa4 = 8.02069252143016600110972019232995528e-02L, /* 0x3ffb4887, 0x0e8b548e, 0x3230b417, 0x11b553b3 */ +pa5 = -1.02729816533435279443621120242391295e-02L, /* 0xbff850a0, 0x041de3ee, 0xd5bca6c9, 0x4ef5f9f2 */ +pa6 = 5.70777694530755634864821094419982095e-03L, /* 0x3ff77610, 0x9b501e10, 0x4c978382, 0x742df68f */ +pa7 = 1.22635150233075521018231779267077071e-03L, /* 0x3ff5417b, 0x0e623682, 0x60327da0, 0x96b9219e */ +pa8 = 5.36100234820204569428412542856666503e-04L, /* 0x3ff41912, 0x27ceb4c1, 0x1d3298ec, 0x84ced627 */ +pa9 = -1.97753571846365167177187858667583165e-04L, /* 0xbff29eb8, 0x23f5bcf3, 0x15c83c46, 0xe4fda98b */ +pa10 = 6.19333039900846970674794789568415105e-05L, /* 0x3ff103c4, 0x60f88e46, 0xc0c9fb02, 0x13cc7fc1 */ +pa11 = -5.40531400436645861492290270311751349e-06L, /* 0xbfed6abe, 0x9665f8a8, 0xdd0ad3ba, 0xe5dc0ee3 */ +qa1 = 9.05041313265490487793231810291907851e-01L, /* 0x3ffecf61, 0x93340222, 0xe9930620, 0xc4e61168 */ +qa2 = 6.79848064708886864767240880834868092e-01L, /* 0x3ffe5c15, 0x0ba858dc, 0xf7900ae9, 0xfea1e09a */ +qa3 = 4.04720609926471677581066689316516445e-01L, /* 0x3ffd9e6f, 0x145e9b00, 0x6d8c1749, 0xd2928623 */ +qa4 = 1.69183273898369996364661075664302225e-01L, /* 0x3ffc5a7c, 0xc2a363c1, 0xd6c19097, 0xef9b4063 */ +qa5 = 7.44476185988067992342479750486764248e-02L, /* 0x3ffb30ef, 0xfc7259ef, 0x1bcbb089, 0x686dd62d */ +qa6 = 2.02981172725892407200420389604788573e-02L, /* 0x3ff94c90, 0x7976cb0e, 0x21e1d36b, 0x0f09ca2b */ +qa7 = 6.94281866271607668268269403102277234e-03L, /* 0x3ff7c701, 0x2b193250, 0xc5d46ecc, 0x374843d8 */ +qa8 = 1.12952275469171559611651594706820034e-03L, /* 0x3ff52818, 0xfd2a7c06, 0xd13e38fd, 0xda4b34f5 */ +qa9 = 3.13736683241992737197226578597710179e-04L, /* 0x3ff348fa, 0x0cb48d18, 0x051f849b, 0x135ccf74 */ +qa10 = 1.17037675204033225470121134087771410e-05L, /* 0x3fee88b6, 0x98f47704, 0xa5d8f8f2, 0xc6422e11 */ +qa11 = 4.61312518293853991439362806880973592e-06L, /* 0x3fed3594, 0xe31db94f, 0x3592b693, 0xed4386b4 */ +qa12 = -1.02158572037456893687737553657431771e-06L; /* 0xbfeb123a, 0xd60d9b1e, 0x1f6fdeb9, 0x7dc8410a */ +/* + * Domain [1.25,2.85715], range ~[-2.922e-37,2.922e-37]: + * |log(x*erfc(x)) + x**2 + 0.5625 - ra(x)/sa(x)| < 2**-121.36 + */ +static const long double +ra0 = -9.86494292470069009555706994426014461e-03L, /* 0xbff84341, 0x239e8709, 0xe941b06a, 0xcb4b6ec5 */ +ra1 = -1.13580436992565640457579040117568870e+00L, /* 0xbfff22c4, 0x133f7c0d, 0x72d5e231, 0x2eb1ee3f */ +ra2 = -4.89744330295291950661185707066921755e+01L, /* 0xc00487cb, 0xa38b4fc2, 0xc136695b, 0xc1df8047 */ +ra3 = -1.10766149300215937173768072715352140e+03L, /* 0xc00914ea, 0x55e6beb3, 0xabc50e07, 0xb6e5664d */ +ra4 = -1.49991031232170934967642795601952100e+04L, /* 0xc00cd4b8, 0xd33243e6, 0xffbf6545, 0x3c57ef6e */ +ra5 = -1.29805749738318462882524181556996692e+05L, /* 0xc00ffb0d, 0xbfeed9b6, 0x5b2a3ff4, 0xe245bd3c */ +ra6 = -7.42828497044940065828871976644647850e+05L, /* 0xc0126ab5, 0x8fe7caca, 0x473352d9, 0xcd4e0c90 */ +ra7 = -2.85637299581890734287995171242421106e+06L, /* 0xc0145cad, 0xa7f76fe7, 0x3e358051, 0x1799f927 */ +ra8 = -7.40674797129824999383748865571026084e+06L, /* 0xc015c412, 0x6fe29c02, 0x298ad158, 0x7d24e45c */ +ra9 = -1.28653420911930973914078724204151759e+07L, /* 0xc016889e, 0x7c2eb0dc, 0x95d5863b, 0x0aa34dc3 */ +ra10 = -1.47198163599330179552932489109452638e+07L, /* 0xc016c136, 0x90b84923, 0xf9bcb497, 0x19bbd0f5 */ +ra11 = -1.07812992258382800318665248311522624e+07L, /* 0xc0164904, 0xe673a113, 0x35d7f079, 0xe13701f3 */ +ra12 = -4.83545565681708642630419905537756076e+06L, /* 0xc0152721, 0xfea094a8, 0x869eb39d, 0x413d6f13 */ +ra13 = -1.23956521201673964822976917356685286e+06L, /* 0xc0132ea0, 0xd3646baa, 0x2fe62b0d, 0xbae5ce85 */ +ra14 = -1.62289333553652417591275333240371812e+05L, /* 0xc0103cf8, 0xaab1e2d6, 0x4c25e014, 0x248d76ab */ +ra15 = -8.82890392601176969729168894389833110e+03L, /* 0xc00c13e7, 0x3b3d8f94, 0x6fbda6f6, 0xe7049a82 */ +ra16 = -1.22591866337261720023681535568334619e+02L, /* 0xc005ea5e, 0x12358891, 0xcfa712c5, 0x77f050d4 */ +sa1 = 6.44508918884710829371852723353794047e+01L, /* 0x400501cd, 0xb69a6c0f, 0x5716de14, 0x47161af6 */ +sa2 = 1.76118475473171481523704824327358534e+03L, /* 0x4009b84b, 0xd305829f, 0xc4c771b0, 0xbf1f7f9b */ +sa3 = 2.69448346969488374857087646131950188e+04L, /* 0x400da503, 0x56bacc05, 0x4fdba68d, 0x2cca27e6 */ +sa4 = 2.56826633369941456778326497384543763e+05L, /* 0x4010f59d, 0x51124428, 0x69c41de6, 0xbd0d5753 */ +sa5 = 1.60647413092257206847700054645905859e+06L, /* 0x40138834, 0xa2184244, 0x557a1bed, 0x68c9d556 */ +sa6 = 6.76963075165099718574753447122393797e+06L, /* 0x40159d2f, 0x7b01b0cc, 0x8bac9e95, 0x5d35d56e */ +sa7 = 1.94295690905361884290986932493647741e+07L, /* 0x40172878, 0xc1172d61, 0x3068501e, 0x2f3c71da */ +sa8 = 3.79774781017759149060839255547073541e+07L, /* 0x401821be, 0xc30d06fe, 0x410563d7, 0x032111fd */ +sa9 = 5.00659831846029484248302236457727397e+07L, /* 0x40187df9, 0x1f97a111, 0xc51d6ac2, 0x4b389793 */ +sa10 = 4.36486287620506484276130525941972541e+07L, /* 0x40184d03, 0x3a618ae0, 0x2a723357, 0xfa45c60a */ +sa11 = 2.43779678791333894255510508253951934e+07L, /* 0x401773fa, 0x6fe10ee2, 0xc467850d, 0xc6b7ff30 */ +sa12 = 8.30732360384443202039372372212966542e+06L, /* 0x4015fb09, 0xee6a5631, 0xdd98de7e, 0x8b00461a */ +sa13 = 1.60160846942050515734192397495105693e+06L, /* 0x40138704, 0x8782bf13, 0x5b8fb315, 0xa898abe5 */ +sa14 = 1.54255505242533291014555153757001825e+05L, /* 0x40102d47, 0xc0abc98e, 0x843c9490, 0xb4352440 */ +sa15 = 5.87949220002375547561467275493888824e+03L, /* 0x400b6f77, 0xe00d21d1, 0xec4d41e8, 0x2f8e1673 */ +sa16 = 4.97272976346793193860385983372237710e+01L; /* 0x40048dd1, 0x816c1b3f, 0x24f540a6, 0x4cfe03cc */ +/* + * Domain [2.85715,9], range ~[-7.886e-37,7.918e-37]: + * |log(x*erfc(x)) + x**2 + 0.5625 - rb(x)/sb(x)| < 2**-120 + */ +static const long double +rb0 = -9.86494292470008707171371994479162369e-3L, /* 0xbff84341, 0x239e86f4, 0x2f57e561, 0xf4469360 */ +rb1 = -1.57047326624110727986326503729442830L, /* 0xbfff920a, 0x8935bf73, 0x8803b894, 0x4656482d */ +rb2 = -1.03228196364885474342132255440317065e2L, /* 0xc0059ce9, 0xac4ed0ff, 0x2cff0ff7, 0x5e70d1ab */ +rb3 = -3.74000570653418227179358710865224376e3L, /* 0xc00ad380, 0x2ebf7835, 0xf6b07ed2, 0x861242f7 */ +rb4 = -8.35435477739098044190860390632813956e4L, /* 0xc00f4657, 0x8c3ae934, 0x3647d7b3, 0x80e76fb7 */ +rb5 = -1.21398672055223642118716640216747152e6L, /* 0xc0132862, 0x2b8761c8, 0x27d18c0f, 0x137c9463 */ +rb6 = -1.17669175877248796101665344873273970e7L, /* 0xc0166719, 0x0b2cea46, 0x81f14174, 0x11602ea5 */ +rb7 = -7.66108006086998253606773064264599615e7L, /* 0xc019243f, 0x3c26f4f0, 0x1cc05241, 0x3b953728 */ +rb8 = -3.32547117558141845968704725353130804e8L, /* 0xc01b3d24, 0x42d8ee26, 0x24ef6f3b, 0x604a8c65 */ +rb9 = -9.41561252426350696802167711221739746e8L, /* 0xc01cc0f8, 0xad23692a, 0x8ddb2310, 0xe9937145 */ +rb10 = -1.67157110805390944549427329626281063e9L, /* 0xc01d8e88, 0x9a903734, 0x09a55fa3, 0xd205c903 */ +rb11 = -1.74339631004410841337645931421427373e9L, /* 0xc01d9fa8, 0x77582d2a, 0xc183b8ab, 0x7e00cb05 */ +rb12 = -9.57655233596934915727573141357471703e8L, /* 0xc01cc8a5, 0x460cc685, 0xd0271fa0, 0x6a70e3da */ +rb13 = -2.26320062731339353035254704082495066e8L, /* 0xc01aafab, 0xd7d76721, 0xc9720e11, 0x6a8bd489 */ +rb14 = -1.42777302996263256686002973851837039e7L, /* 0xc016b3b8, 0xc499689f, 0x2b88d965, 0xc32414f9 */ +sb1 = 1.08512869705594540211033733976348506e2L, /* 0x4005b20d, 0x2db7528d, 0x00d20dcb, 0x858f6191 */ +sb2 = 5.02757713761390460534494530537572834e3L, /* 0x400b3a39, 0x3bf4a690, 0x3025d28d, 0xfd40a891 */ +sb3 = 1.31019107205412870059331647078328430e5L, /* 0x400fffcb, 0x1b71d05e, 0x3b28361d, 0x2a3c3690 */ +sb4 = 2.13021555152296846166736757455018030e6L, /* 0x40140409, 0x3c6984df, 0xc4491d7c, 0xb04aa08d */ +sb5 = 2.26649105281820861953868568619768286e7L, /* 0x401759d6, 0xce8736f0, 0xf28ad037, 0x2a901e0c */ +sb6 = 1.61071939490875921812318684143076081e8L, /* 0x401a3338, 0x686fb541, 0x6bd27d06, 0x4f95c9ac */ +sb7 = 7.66895673844301852676056750497991966e8L, /* 0x401c6daf, 0x31cec121, 0x54699126, 0x4bd9bf9e */ +sb8 = 2.41884450436101936436023058196042526e9L, /* 0x401e2059, 0x46b0b8d7, 0x87b64cbf, 0x78bc296d */ +sb9 = 4.92403055884071695093305291535107666e9L, /* 0x401f257e, 0xbe5ed739, 0x39e17346, 0xcadd2e55 */ +sb10 = 6.18627786365587486459633615573786416e9L, /* 0x401f70bb, 0x1be7a7e7, 0x6a45b5ae, 0x607c70f0 */ +sb11 = 4.45898013426501378097430226324743199e9L, /* 0x401f09c6, 0xa32643d7, 0xf1724620, 0x9ea46c32 */ +sb12 = 1.63006115763329848117160344854224975e9L, /* 0x401d84a3, 0x0996887f, 0x65a4f43b, 0x978c1d74 */ +sb13 = 2.39216717012421697446304015847567721e8L, /* 0x401ac845, 0x09a065c2, 0x30095da7, 0x9d72d6ae */ +sb14 = 7.84837329009278694937250358810225609e6L; /* 0x4015df06, 0xd5290e15, 0x63031fac, 0x4d9c894c */ +/* + * Domain [9,108], range ~[-5.324e-38,5.340e-38]: + * |log(x*erfc(x)) + x**2 + 0.5625 - r(x)/s(x)| < 2**-124 + */ +static const long double +rc0 = -9.86494292470008707171367567652935673e-3L, /* 0xbff84341, 0x239e86f4, 0x2f57e55b, 0x1aa10fd3 */ +rc1 = -1.26229447747315096406518846411562266L, /* 0xbfff4325, 0xbb1aab28, 0xda395cd9, 0xfb861c15 */ +rc2 = -6.13742634438922591780742637728666162e1L, /* 0xc004eafe, 0x7dd51cd8, 0x3c7c5928, 0x751e50cf */ +rc3 = -1.50455835478908280402912854338421517e3L, /* 0xc0097823, 0xbc15b9ab, 0x3d60745c, 0x523e80a5 */ +rc4 = -2.04415631865861549920184039902945685e4L, /* 0xc00d3f66, 0x40b3fc04, 0x5388f2ec, 0xb009e1f0 */ +rc5 = -1.57625662981714582753490610560037638e5L, /* 0xc01033dc, 0xd4dc95b6, 0xfd4da93b, 0xf355b4a9 */ +rc6 = -6.73473451616752528402917538033283794e5L, /* 0xc01248d8, 0x2e73a4f9, 0xcded49c5, 0xfa3bfeb7 */ +rc7 = -1.47433165421387483167186683764364857e6L, /* 0xc01367f1, 0xba77a8f7, 0xcfdd0dbb, 0x25d554b3 */ +rc8 = -1.38811981807868828563794929997744139e6L, /* 0xc01352e5, 0x7d16d9ad, 0xbbdcbf38, 0x38fbc5ea */ +rc9 = -3.59659700530831825640766479698155060e5L, /* 0xc0115f3a, 0xecd57f45, 0x21f8ad6c, 0x910a5958 */ +sc1 = 7.72730753022908298637508998072635696e1L, /* 0x40053517, 0xa10d52bc, 0xdabb55b6, 0xbd0328cd */ +sc2 = 2.36825757341694050500333261769082182e3L, /* 0x400a2808, 0x3e0a9b42, 0x82977842, 0x9c5de29e */ +sc3 = 3.72210540173034735352888847134073099e4L, /* 0x400e22ca, 0x1ba827ef, 0xac8390d7, 0x1fc39a41 */ +sc4 = 3.24136032646418336712461033591393412e5L, /* 0x40113c8a, 0x0216e100, 0xc59d1e44, 0xf0e68d9d */ +sc5 = 1.57836135851134393802505823370009175e6L, /* 0x40138157, 0x95bc7664, 0x17575961, 0xdbe58eeb */ +sc6 = 4.12881981392063738026679089714182355e6L, /* 0x4014f801, 0x9e82e8d2, 0xb8b3a70e, 0xfd84185d */ +sc7 = 5.24438427289213488410596395361544142e6L, /* 0x40154017, 0x81177109, 0x2aa6c3b0, 0x1f106625 */ +sc8 = 2.59909544563616121735963429710382149e6L, /* 0x40143d45, 0xbb90a9b1, 0x12bf9390, 0xa827a700 */ +sc9 = 2.80930665169282501639651995082335693e5L; /* 0x40111258, 0xaa92222e, 0xa97e3216, 0xa237fa6c */ + +long double +erfl(long double x) +{ + long double ax,R,S,P,Q,s,y,z,r; + uint64_t lx, llx; + int32_t i; + uint16_t hx; + + EXTRACT_LDBL128_WORDS(hx, lx, llx, x); + + if((hx & 0x7fff) == 0x7fff) { /* erfl(nan)=nan */ + i = (hx>>15)<<1; + return (1-i)+one/x; /* erfl(+-inf)=+-1 */ + } + + ax = fabsl(x); + if(ax < 0.84375) { + if(ax < 0x1p-40L) { + if(ax < 0x1p-16373L) + return (8*x+efx8*x)/8; /* avoid spurious underflow */ + return x + efx*x; + } + z = x*x; + r = pp0+z*(pp1+z*(pp2+z*(pp3+z*(pp4+z*(pp5+z*(pp6+z*(pp7+ + z*(pp8+z*pp9)))))))); + s = one+z*(qq1+z*(qq2+z*(qq3+z*(qq4+z*(qq5+z*(qq6+z*(qq7+ + z*(qq8+z*qq9)))))))); + y = r/s; + return x + x*y; + } + if(ax < 1.25) { + s = ax-one; + P = pa0+s*(pa1+s*(pa2+s*(pa3+s*(pa4+s*(pa5+s*(pa6+s*(pa7+ + s*(pa8+s*(pa9+s*(pa10+s*pa11)))))))))); + Q = one+s*(qa1+s*(qa2+s*(qa3+s*(qa4+s*(qa5+s*(qa6+s*(qa7+ + s*(qa8+s*(qa9+s*(qa10+s*(qa11+s*qa12))))))))))); + if(x>=0) return (erx + P/Q); else return (-erx - P/Q); + } + if (ax >= 9) { /* inf>|x|>= 9 */ + if(x>=0) return (one-tiny); else return (tiny-one); + } + s = one/(ax*ax); + if(ax < 2.85715) { /* |x| < 2.85715 */ + R=ra0+s*(ra1+s*(ra2+s*(ra3+s*(ra4+s*(ra5+s*(ra6+s*(ra7+ + s*(ra8+s*(ra9+s*(ra10+s*(ra11+s*(ra12+s*(ra13+s*(ra14+ + s*(ra15+s*ra16))))))))))))))); + S=one+s*(sa1+s*(sa2+s*(sa3+s*(sa4+s*(sa5+s*(sa6+s*(sa7+ + s*(sa8+s*(sa9+s*(sa10+s*(sa11+s*(sa12+s*(sa13+s*(sa14+ + s*(sa15+s*sa16))))))))))))))); + } else { /* |x| >= 2.85715 */ + R=rb0+s*(rb1+s*(rb2+s*(rb3+s*(rb4+s*(rb5+s*(rb6+s*(rb7+ + s*(rb8+s*(rb9+s*(rb10+s*(rb11+s*(rb12+s*(rb13+ + s*rb14))))))))))))); + S=one+s*(sb1+s*(sb2+s*(sb3+s*(sb4+s*(sb5+s*(sb6+s*(sb7+ + s*(sb8+s*(sb9+s*(sb10+s*(sb11+s*(sb12+s*(sb13+ + s*sb14))))))))))))); + } + z = (float)ax; + r = expl(-z*z-0.5625)*expl((z-ax)*(z+ax)+R/S); + if(x>=0) return (one-r/ax); else return (r/ax-one); +} + +long double +erfcl(long double x) +{ + long double ax,R,S,P,Q,s,y,z,r; + uint64_t lx, llx; + uint16_t hx; + + EXTRACT_LDBL128_WORDS(hx, lx, llx, x); + + if((hx & 0x7fff) == 0x7fff) { /* erfcl(nan)=nan */ + /* erfcl(+-inf)=0,2 */ + return ((hx>>15)<<1)+one/x; + } + + ax = fabsl(x); + if(ax < 0.84375L) { + if(ax < 0x1p-34L) + return one-x; + z = x*x; + r = pp0+z*(pp1+z*(pp2+z*(pp3+z*(pp4+z*(pp5+z*(pp6+z*(pp7+ + z*(pp8+z*pp9)))))))); + s = one+z*(qq1+z*(qq2+z*(qq3+z*(qq4+z*(qq5+z*(qq6+z*(qq7+ + z*(qq8+z*qq9)))))))); + y = r/s; + if(ax < 0.25L) { /* x<1/4 */ + return one-(x+x*y); + } else { + r = x*y; + r += (x-half); + return half - r; + } + } + if(ax < 1.25L) { + s = ax-one; + P = pa0+s*(pa1+s*(pa2+s*(pa3+s*(pa4+s*(pa5+s*(pa6+s*(pa7+ + s*(pa8+s*(pa9+s*(pa10+s*pa11)))))))))); + Q = one+s*(qa1+s*(qa2+s*(qa3+s*(qa4+s*(qa5+s*(qa6+s*(qa7+ + s*(qa8+s*(qa9+s*(qa10+s*(qa11+s*qa12))))))))))); + if(x>=0) { + z = one-erx; return z - P/Q; + } else { + z = erx+P/Q; return one+z; + } + } + + if(ax < 108) { /* |x| < 108 */ + s = one/(ax*ax); + if(ax < 2.85715) { /* |x| < 2.85715 */ + R=ra0+s*(ra1+s*(ra2+s*(ra3+s*(ra4+s*(ra5+s*(ra6+s*(ra7+ + s*(ra8+s*(ra9+s*(ra10+s*(ra11+s*(ra12+s*(ra13+s*(ra14+ + s*(ra15+s*ra16))))))))))))))); + S=one+s*(sa1+s*(sa2+s*(sa3+s*(sa4+s*(sa5+s*(sa6+s*(sa7+ + s*(sa8+s*(sa9+s*(sa10+s*(sa11+s*(sa12+s*(sa13+s*(sa14+ + s*(sa15+s*sa16))))))))))))))); + } else if(ax < 9) { + R=rb0+s*(rb1+s*(rb2+s*(rb3+s*(rb4+s*(rb5+s*(rb6+s*(rb7+ + s*(rb8+s*(rb9+s*(rb10+s*(rb11+s*(rb12+s*(rb13+ + s*rb14))))))))))))); + S=one+s*(sb1+s*(sb2+s*(sb3+s*(sb4+s*(sb5+s*(sb6+s*(sb7+ + s*(sb8+s*(sb9+s*(sb10+s*(sb11+s*(sb12+s*(sb13+ + s*sb14))))))))))))); + } else { + if(x < -9) return two-tiny; /* x < -9 */ + R=rc0+s*(rc1+s*(rc2+s*(rc3+s*(rc4+s*(rc5+s*(rc6+s*(rc7+ + s*(rc8+s*rc9)))))))); + S=one+s*(sc1+s*(sc2+s*(sc3+s*(sc4+s*(sc5+s*(sc6+s*(sc7+ + s*(sc8+s*sc9)))))))); + } + z = (float)ax; + r = expl(-z*z-0.5625)*expl((z-ax)*(z+ax)+R/S); + if(x>0) return r/ax; else return two-r/ax; + } else { + if(x>0) return tiny*tiny; else return two-tiny; + } +} diff --git a/libm/upstream-freebsd/lib/msun/ld128/s_exp2l.c b/libm/upstream-freebsd/lib/msun/ld128/s_exp2l.c index 5ed514c97..5afa37e58 100644 --- a/libm/upstream-freebsd/lib/msun/ld128/s_exp2l.c +++ b/libm/upstream-freebsd/lib/msun/ld128/s_exp2l.c @@ -369,7 +369,7 @@ exp2l(long double x) || u.xbits.manl != 0 || (hx & 0x8000) == 0) return (x + x); /* x is NaN or +Inf */ - else + else return (0.0); /* x is -Inf */ } if (x >= 16384) diff --git a/libm/upstream-freebsd/lib/msun/ld128/s_expl.c b/libm/upstream-freebsd/lib/msun/ld128/s_expl.c index 176c93255..a6a967675 100644 --- a/libm/upstream-freebsd/lib/msun/ld128/s_expl.c +++ b/libm/upstream-freebsd/lib/msun/ld128/s_expl.c @@ -38,16 +38,15 @@ __FBSDID("$FreeBSD$"); #include "fpmath.h" #include "math.h" #include "math_private.h" +#include "k_expl.h" -#define INTERVALS 128 -#define LOG2_INTERVALS 7 -#define BIAS (LDBL_MAX_EXP - 1) +/* XXX Prevent compilers from erroneously constant folding these: */ +static const volatile long double +huge = 0x1p10000L, +tiny = 0x1p-10000L; static const long double -huge = 0x1p10000L, twom10000 = 0x1p-10000L; -/* XXX Prevent gcc from erroneously constant folding this: */ -static volatile const long double tiny = 0x1p-10000L; static const long double /* log(2**16384 - 0.5) rounded towards zero: */ @@ -56,184 +55,16 @@ o_threshold = 11356.523406294143949491931077970763428L, /* log(2**(-16381-64-1)) rounded towards zero: */ u_threshold = -11433.462743336297878837243843452621503L; -static const double -/* - * ln2/INTERVALS = L1+L2 (hi+lo decomposition for multiplication). L1 must - * have at least 22 (= log2(|LDBL_MIN_EXP-extras|) + log2(INTERVALS)) lowest - * bits zero so that multiplication of it by n is exact. - */ -INV_L = 1.8466496523378731e+2, /* 0x171547652b82fe.0p-45 */ -L2 = -1.0253670638894731e-29; /* -0x1.9ff0342542fc3p-97 */ -static const long double -/* 0x1.62e42fefa39ef35793c768000000p-8 */ -L1 = 5.41521234812457272982212595914567508e-3L; - -static const long double -/* - * Domain [-0.002708, 0.002708], range ~[-2.4021e-38, 2.4234e-38]: - * |exp(x) - p(x)| < 2**-124.9 - * (0.002708 is ln2/(2*INTERVALS) rounded up a little). - */ -A2 = 0.5, -A3 = 1.66666666666666666666666666651085500e-1L, -A4 = 4.16666666666666666666666666425885320e-2L, -A5 = 8.33333333333333333334522877160175842e-3L, -A6 = 1.38888888888888888889971139751596836e-3L; - -static const double -A7 = 1.9841269841269471e-4, -A8 = 2.4801587301585284e-5, -A9 = 2.7557324277411234e-6, -A10 = 2.7557333722375072e-7; - -static const struct { - /* - * hi must be rounded to at most 106 bits so that multiplication - * by r1 in expm1l() is exact, but it is rounded to 88 bits due to - * historical accidents. - */ - long double hi; - long double lo; -} tbl[INTERVALS] = { - 0x1p0L, 0x0p0L, - 0x1.0163da9fb33356d84a66aep0L, 0x3.36dcdfa4003ec04c360be2404078p-92L, - 0x1.02c9a3e778060ee6f7cacap0L, 0x4.f7a29bde93d70a2cabc5cb89ba10p-92L, - 0x1.04315e86e7f84bd738f9a2p0L, 0xd.a47e6ed040bb4bfc05af6455e9b8p-96L, - 0x1.059b0d31585743ae7c548ep0L, 0xb.68ca417fe53e3495f7df4baf84a0p-92L, - 0x1.0706b29ddf6ddc6dc403a8p0L, 0x1.d87b27ed07cb8b092ac75e311753p-88L, - 0x1.0874518759bc808c35f25cp0L, 0x1.9427fa2b041b2d6829d8993a0d01p-88L, - 0x1.09e3ecac6f3834521e060cp0L, 0x5.84d6b74ba2e023da730e7fccb758p-92L, - 0x1.0b5586cf9890f6298b92b6p0L, 0x1.1842a98364291408b3ceb0a2a2bbp-88L, - 0x1.0cc922b7247f7407b705b8p0L, 0x9.3dc5e8aac564e6fe2ef1d431fd98p-92L, - 0x1.0e3ec32d3d1a2020742e4ep0L, 0x1.8af6a552ac4b358b1129e9f966a4p-88L, - 0x1.0fb66affed31af232091dcp0L, 0x1.8a1426514e0b627bda694a400a27p-88L, - 0x1.11301d0125b50a4ebbf1aep0L, 0xd.9318ceac5cc47ab166ee57427178p-92L, - 0x1.12abdc06c31cbfb92bad32p0L, 0x4.d68e2f7270bdf7cedf94eb1cb818p-92L, - 0x1.1429aaea92ddfb34101942p0L, 0x1.b2586d01844b389bea7aedd221d4p-88L, - 0x1.15a98c8a58e512480d573cp0L, 0x1.d5613bf92a2b618ee31b376c2689p-88L, - 0x1.172b83c7d517adcdf7c8c4p0L, 0x1.0eb14a792035509ff7d758693f24p-88L, - 0x1.18af9388c8de9bbbf70b9ap0L, 0x3.c2505c97c0102e5f1211941d2840p-92L, - 0x1.1a35beb6fcb753cb698f68p0L, 0x1.2d1c835a6c30724d5cfae31b84e5p-88L, - 0x1.1bbe084045cd39ab1e72b4p0L, 0x4.27e35f9acb57e473915519a1b448p-92L, - 0x1.1d4873168b9aa7805b8028p0L, 0x9.90f07a98b42206e46166cf051d70p-92L, - 0x1.1ed5022fcd91cb8819ff60p0L, 0x1.121d1e504d36c47474c9b7de6067p-88L, - 0x1.2063b88628cd63b8eeb028p0L, 0x1.50929d0fc487d21c2b84004264dep-88L, - 0x1.21f49917ddc962552fd292p0L, 0x9.4bdb4b61ea62477caa1dce823ba0p-92L, - 0x1.2387a6e75623866c1fadb0p0L, 0x1.c15cb593b0328566902df69e4de2p-88L, - 0x1.251ce4fb2a63f3582ab7dep0L, 0x9.e94811a9c8afdcf796934bc652d0p-92L, - 0x1.26b4565e27cdd257a67328p0L, 0x1.d3b249dce4e9186ddd5ff44e6b08p-92L, - 0x1.284dfe1f5638096cf15cf0p0L, 0x3.ca0967fdaa2e52d7c8106f2e262cp-92L, - 0x1.29e9df51fdee12c25d15f4p0L, 0x1.a24aa3bca890ac08d203fed80a07p-88L, - 0x1.2b87fd0dad98ffddea4652p0L, 0x1.8fcab88442fdc3cb6de4519165edp-88L, - 0x1.2d285a6e4030b40091d536p0L, 0xd.075384589c1cd1b3e4018a6b1348p-92L, - 0x1.2ecafa93e2f5611ca0f45cp0L, 0x1.523833af611bdcda253c554cf278p-88L, - 0x1.306fe0a31b7152de8d5a46p0L, 0x3.05c85edecbc27343629f502f1af2p-92L, - 0x1.32170fc4cd8313539cf1c2p0L, 0x1.008f86dde3220ae17a005b6412bep-88L, - 0x1.33c08b26416ff4c9c8610cp0L, 0x1.96696bf95d1593039539d94d662bp-88L, - 0x1.356c55f929ff0c94623476p0L, 0x3.73af38d6d8d6f9506c9bbc93cbc0p-92L, - 0x1.371a7373aa9caa7145502ep0L, 0x1.4547987e3e12516bf9c699be432fp-88L, - 0x1.38cae6d05d86585a9cb0d8p0L, 0x1.bed0c853bd30a02790931eb2e8f0p-88L, - 0x1.3a7db34e59ff6ea1bc9298p0L, 0x1.e0a1d336163fe2f852ceeb134067p-88L, - 0x1.3c32dc313a8e484001f228p0L, 0xb.58f3775e06ab66353001fae9fca0p-92L, - 0x1.3dea64c12342235b41223ep0L, 0x1.3d773fba2cb82b8244267c54443fp-92L, - 0x1.3fa4504ac801ba0bf701aap0L, 0x4.1832fb8c1c8dbdff2c49909e6c60p-92L, - 0x1.4160a21f72e29f84325b8ep0L, 0x1.3db61fb352f0540e6ba05634413ep-88L, - 0x1.431f5d950a896dc7044394p0L, 0x1.0ccec81e24b0caff7581ef4127f7p-92L, - 0x1.44e086061892d03136f408p0L, 0x1.df019fbd4f3b48709b78591d5cb5p-88L, - 0x1.46a41ed1d005772512f458p0L, 0x1.229d97df404ff21f39c1b594d3a8p-88L, - 0x1.486a2b5c13cd013c1a3b68p0L, 0x1.062f03c3dd75ce8757f780e6ec99p-88L, - 0x1.4a32af0d7d3de672d8bcf4p0L, 0x6.f9586461db1d878b1d148bd3ccb8p-92L, - 0x1.4bfdad5362a271d4397afep0L, 0xc.42e20e0363ba2e159c579f82e4b0p-92L, - 0x1.4dcb299fddd0d63b36ef1ap0L, 0x9.e0cc484b25a5566d0bd5f58ad238p-92L, - 0x1.4f9b2769d2ca6ad33d8b68p0L, 0x1.aa073ee55e028497a329a7333dbap-88L, - 0x1.516daa2cf6641c112f52c8p0L, 0x4.d822190e718226177d7608d20038p-92L, - 0x1.5342b569d4f81df0a83c48p0L, 0x1.d86a63f4e672a3e429805b049465p-88L, - 0x1.551a4ca5d920ec52ec6202p0L, 0x4.34ca672645dc6c124d6619a87574p-92L, - 0x1.56f4736b527da66ecb0046p0L, 0x1.64eb3c00f2f5ab3d801d7cc7272dp-88L, - 0x1.58d12d497c7fd252bc2b72p0L, 0x1.43bcf2ec936a970d9cc266f0072fp-88L, - 0x1.5ab07dd48542958c930150p0L, 0x1.91eb345d88d7c81280e069fbdb63p-88L, - 0x1.5c9268a5946b701c4b1b80p0L, 0x1.6986a203d84e6a4a92f179e71889p-88L, - 0x1.5e76f15ad21486e9be4c20p0L, 0x3.99766a06548a05829e853bdb2b52p-92L, - 0x1.605e1b976dc08b076f592ap0L, 0x4.86e3b34ead1b4769df867b9c89ccp-92L, - 0x1.6247eb03a5584b1f0fa06ep0L, 0x1.d2da42bb1ceaf9f732275b8aef30p-88L, - 0x1.6434634ccc31fc76f8714cp0L, 0x4.ed9a4e41000307103a18cf7a6e08p-92L, - 0x1.66238825522249127d9e28p0L, 0x1.b8f314a337f4dc0a3adf1787ff74p-88L, - 0x1.68155d44ca973081c57226p0L, 0x1.b9f32706bfe4e627d809a85dcc66p-88L, - 0x1.6a09e667f3bcc908b2fb12p0L, 0x1.66ea957d3e3adec17512775099dap-88L, - 0x1.6c012750bdabeed76a9980p0L, 0xf.4f33fdeb8b0ecd831106f57b3d00p-96L, - 0x1.6dfb23c651a2ef220e2cbep0L, 0x1.bbaa834b3f11577ceefbe6c1c411p-92L, - 0x1.6ff7df9519483cf87e1b4ep0L, 0x1.3e213bff9b702d5aa477c12523cep-88L, - 0x1.71f75e8ec5f73dd2370f2ep0L, 0xf.0acd6cb434b562d9e8a20adda648p-92L, - 0x1.73f9a48a58173bd5c9a4e6p0L, 0x8.ab1182ae217f3a7681759553e840p-92L, - 0x1.75feb564267c8bf6e9aa32p0L, 0x1.a48b27071805e61a17b954a2dad8p-88L, - 0x1.780694fde5d3f619ae0280p0L, 0x8.58b2bb2bdcf86cd08e35fb04c0f0p-92L, - 0x1.7a11473eb0186d7d51023ep0L, 0x1.6cda1f5ef42b66977960531e821bp-88L, - 0x1.7c1ed0130c1327c4933444p0L, 0x1.937562b2dc933d44fc828efd4c9cp-88L, - 0x1.7e2f336cf4e62105d02ba0p0L, 0x1.5797e170a1427f8fcdf5f3906108p-88L, - 0x1.80427543e1a11b60de6764p0L, 0x9.a354ea706b8e4d8b718a672bf7c8p-92L, - 0x1.82589994cce128acf88afap0L, 0xb.34a010f6ad65cbbac0f532d39be0p-92L, - 0x1.8471a4623c7acce52f6b96p0L, 0x1.c64095370f51f48817914dd78665p-88L, - 0x1.868d99b4492ec80e41d90ap0L, 0xc.251707484d73f136fb5779656b70p-92L, - 0x1.88ac7d98a669966530bcdep0L, 0x1.2d4e9d61283ef385de170ab20f96p-88L, - 0x1.8ace5422aa0db5ba7c55a0p0L, 0x1.92c9bb3e6ed61f2733304a346d8fp-88L, - 0x1.8cf3216b5448bef2aa1cd0p0L, 0x1.61c55d84a9848f8c453b3ca8c946p-88L, - 0x1.8f1ae991577362b982745cp0L, 0x7.2ed804efc9b4ae1458ae946099d4p-92L, - 0x1.9145b0b91ffc588a61b468p0L, 0x1.f6b70e01c2a90229a4c4309ea719p-88L, - 0x1.93737b0cdc5e4f4501c3f2p0L, 0x5.40a22d2fc4af581b63e8326efe9cp-92L, - 0x1.95a44cbc8520ee9b483694p0L, 0x1.a0fc6f7c7d61b2b3a22a0eab2cadp-88L, - 0x1.97d829fde4e4f8b9e920f8p0L, 0x1.1e8bd7edb9d7144b6f6818084cc7p-88L, - 0x1.9a0f170ca07b9ba3109b8cp0L, 0x4.6737beb19e1eada6825d3c557428p-92L, - 0x1.9c49182a3f0901c7c46b06p0L, 0x1.1f2be58ddade50c217186c90b457p-88L, - 0x1.9e86319e323231824ca78ep0L, 0x6.4c6e010f92c082bbadfaf605cfd4p-92L, - 0x1.a0c667b5de564b29ada8b8p0L, 0xc.ab349aa0422a8da7d4512edac548p-92L, - 0x1.a309bec4a2d3358c171f76p0L, 0x1.0daad547fa22c26d168ea762d854p-88L, - 0x1.a5503b23e255c8b424491cp0L, 0xa.f87bc8050a405381703ef7caff50p-92L, - 0x1.a799e1330b3586f2dfb2b0p0L, 0x1.58f1a98796ce8908ae852236ca94p-88L, - 0x1.a9e6b5579fdbf43eb243bcp0L, 0x1.ff4c4c58b571cf465caf07b4b9f5p-88L, - 0x1.ac36bbfd3f379c0db966a2p0L, 0x1.1265fc73e480712d20f8597a8e7bp-88L, - 0x1.ae89f995ad3ad5e8734d16p0L, 0x1.73205a7fbc3ae675ea440b162d6cp-88L, - 0x1.b0e07298db66590842acdep0L, 0x1.c6f6ca0e5dcae2aafffa7a0554cbp-88L, - 0x1.b33a2b84f15faf6bfd0e7ap0L, 0x1.d947c2575781dbb49b1237c87b6ep-88L, - 0x1.b59728de559398e3881110p0L, 0x1.64873c7171fefc410416be0a6525p-88L, - 0x1.b7f76f2fb5e46eaa7b081ap0L, 0xb.53c5354c8903c356e4b625aacc28p-92L, - 0x1.ba5b030a10649840cb3c6ap0L, 0xf.5b47f297203757e1cc6eadc8bad0p-92L, - 0x1.bcc1e904bc1d2247ba0f44p0L, 0x1.b3d08cd0b20287092bd59be4ad98p-88L, - 0x1.bf2c25bd71e088408d7024p0L, 0x1.18e3449fa073b356766dfb568ff4p-88L, - 0x1.c199bdd85529c2220cb12ap0L, 0x9.1ba6679444964a36661240043970p-96L, - 0x1.c40ab5fffd07a6d14df820p0L, 0xf.1828a5366fd387a7bdd54cdf7300p-92L, - 0x1.c67f12e57d14b4a2137fd2p0L, 0xf.2b301dd9e6b151a6d1f9d5d5f520p-96L, - 0x1.c8f6d9406e7b511acbc488p0L, 0x5.c442ddb55820171f319d9e5076a8p-96L, - 0x1.cb720dcef90691503cbd1ep0L, 0x9.49db761d9559ac0cb6dd3ed599e0p-92L, - 0x1.cdf0b555dc3f9c44f8958ep0L, 0x1.ac51be515f8c58bdfb6f5740a3a4p-88L, - 0x1.d072d4a07897b8d0f22f20p0L, 0x1.a158e18fbbfc625f09f4cca40874p-88L, - 0x1.d2f87080d89f18ade12398p0L, 0x9.ea2025b4c56553f5cdee4c924728p-92L, - 0x1.d5818dcfba48725da05aeap0L, 0x1.66e0dca9f589f559c0876ff23830p-88L, - 0x1.d80e316c98397bb84f9d04p0L, 0x8.805f84bec614de269900ddf98d28p-92L, - 0x1.da9e603db3285708c01a5ap0L, 0x1.6d4c97f6246f0ec614ec95c99392p-88L, - 0x1.dd321f301b4604b695de3cp0L, 0x6.30a393215299e30d4fb73503c348p-96L, - 0x1.dfc97337b9b5eb968cac38p0L, 0x1.ed291b7225a944efd5bb5524b927p-88L, - 0x1.e264614f5a128a12761fa0p0L, 0x1.7ada6467e77f73bf65e04c95e29dp-88L, - 0x1.e502ee78b3ff6273d13014p0L, 0x1.3991e8f49659e1693be17ae1d2f9p-88L, - 0x1.e7a51fbc74c834b548b282p0L, 0x1.23786758a84f4956354634a416cep-88L, - 0x1.ea4afa2a490d9858f73a18p0L, 0xf.5db301f86dea20610ceee13eb7b8p-92L, - 0x1.ecf482d8e67f08db0312fap0L, 0x1.949cef462010bb4bc4ce72a900dfp-88L, - 0x1.efa1bee615a27771fd21a8p0L, 0x1.2dac1f6dd5d229ff68e46f27e3dfp-88L, - 0x1.f252b376bba974e8696fc2p0L, 0x1.6390d4c6ad5476b5162f40e1d9a9p-88L, - 0x1.f50765b6e4540674f84b76p0L, 0x2.862baff99000dfc4352ba29b8908p-92L, - 0x1.f7bfdad9cbe138913b4bfep0L, 0x7.2bd95c5ce7280fa4d2344a3f5618p-92L, - 0x1.fa7c1819e90d82e90a7e74p0L, 0xb.263c1dc060c36f7650b4c0f233a8p-92L, - 0x1.fd3c22b8f71f10975ba4b2p0L, 0x1.2bcf3a5e12d269d8ad7c1a4a8875p-88L -}; - long double expl(long double x) { - union IEEEl2bits u, v; - long double q, r, r1, t, twopk, twopkp10000; - double dr, fn, r2; - int k, n, n2; + union IEEEl2bits u; + long double hi, lo, t, twopk; + int k; uint16_t hx, ix; + DOPRINT_START(&x); + /* Filter out exceptional cases. */ u.e = x; hx = u.xbits.expsign; @@ -241,60 +72,33 @@ expl(long double x) if (ix >= BIAS + 13) { /* |x| >= 8192 or x is NaN */ if (ix == BIAS + LDBL_MAX_EXP) { if (hx & 0x8000) /* x is -Inf or -NaN */ - return (-1 / x); - return (x + x); /* x is +Inf or +NaN */ + RETURNP(-1 / x); + RETURNP(x + x); /* x is +Inf or +NaN */ } if (x > o_threshold) - return (huge * huge); + RETURNP(huge * huge); if (x < u_threshold) - return (tiny * tiny); + RETURNP(tiny * tiny); } else if (ix < BIAS - 114) { /* |x| < 0x1p-114 */ - return (1 + x); /* 1 with inexact iff x != 0 */ + RETURN2P(1, x); /* 1 with inexact iff x != 0 */ } ENTERI(); - /* Reduce x to (k*ln2 + endpoint[n2] + r1 + r2). */ - /* Use a specialized rint() to get fn. Assume round-to-nearest. */ - /* XXX assume no extra precision for the additions, as for trig fns. */ - /* XXX this set of comments is now quadruplicated. */ - fn = (double)x * INV_L + 0x1.8p52 - 0x1.8p52; -#if defined(HAVE_EFFICIENT_IRINT) - n = irint(fn); -#else - n = (int)fn; -#endif - n2 = (unsigned)n % INTERVALS; - k = n >> LOG2_INTERVALS; - r1 = x - fn * L1; - r2 = fn * -L2; - r = r1 + r2; - - /* Prepare scale factors. */ - /* XXX sparc64 multiplication is so slow that scalbnl() is faster. */ - v.e = 1; - if (k >= LDBL_MIN_EXP) { - v.xbits.expsign = BIAS + k; - twopk = v.e; - } else { - v.xbits.expsign = BIAS + k + 10000; - twopkp10000 = v.e; - } - - /* Evaluate expl(endpoint[n2] + r1 + r2) = tbl[n2] * expl(r1 + r2). */ - dr = r; - q = r2 + r * r * (A2 + r * (A3 + r * (A4 + r * (A5 + r * (A6 + - dr * (A7 + dr * (A8 + dr * (A9 + dr * A10)))))))); - t = tbl[n2].lo + tbl[n2].hi; - t = tbl[n2].lo + t * (q + r1) + tbl[n2].hi; + twopk = 1; + __k_expl(x, &hi, &lo, &k); + t = SUM2P(hi, lo); /* Scale by 2**k. */ + /* XXX sparc64 multiplication is so slow that scalbnl() is faster. */ if (k >= LDBL_MIN_EXP) { if (k == LDBL_MAX_EXP) RETURNI(t * 2 * 0x1p16383L); + SET_LDBL_EXPSIGN(twopk, BIAS + k); RETURNI(t * twopk); } else { - RETURNI(t * twopkp10000 * twom10000); + SET_LDBL_EXPSIGN(twopk, BIAS + k + 10000); + RETURNI(t * twopk * twom10000); } } @@ -312,6 +116,12 @@ expl(long double x) * Setting T3 to 0 would require the |x| < 0x1p-113 condition to appear * in both subintervals, so set T3 = 2**-5, which places the condition * into the [T1, T3] interval. + * + * XXX we now do this more to (partially) balance the number of terms + * in the C and D polys than to avoid checking the condition in both + * intervals. + * + * XXX these micro-optimizations are excessive. */ static const double T1 = -0.1659, /* ~-30.625/128 * log(2) */ @@ -321,6 +131,12 @@ T3 = 0.03125; /* * Domain [-0.1659, 0.03125], range ~[2.9134e-44, 1.8404e-37]: * |(exp(x)-1-x-x**2/2)/x - p(x)| < 2**-122.03 + * + * XXX none of the long double C or D coeffs except C10 is correctly printed. + * If you re-print their values in %.35Le format, the result is always + * different. For example, the last 2 digits in C3 should be 59, not 67. + * 67 is apparently from rounding an extra-precision value to 36 decimal + * places. */ static const long double C3 = 1.66666666666666666666666666666666667e-1L, @@ -335,6 +151,13 @@ C11 = 2.50521083854417203619031960151253944e-8L, C12 = 2.08767569878679576457272282566520649e-9L, C13 = 1.60590438367252471783548748824255707e-10L; +/* + * XXX this has 1 more coeff than needed. + * XXX can start the double coeffs but not the double mults at C10. + * With my coeffs (C10-C17 double; s = best_s): + * Domain [-0.1659, 0.03125], range ~[-1.1976e-37, 1.1976e-37]: + * |(exp(x)-1-x-x**2/2)/x - p(x)| ~< 2**-122.65 + */ static const double C14 = 1.1470745580491932e-11, /* 0x1.93974a81dae30p-37 */ C15 = 7.6471620181090468e-13, /* 0x1.ae7f3820adab1p-41 */ @@ -359,6 +182,13 @@ D11 = 2.50521083855084570046480450935267433e-8L, D12 = 2.08767569819738524488686318024854942e-9L, D13 = 1.60590442297008495301927448122499313e-10L; +/* + * XXX this has 1 more coeff than needed. + * XXX can start the double coeffs but not the double mults at D11. + * With my coeffs (D11-D16 double): + * Domain [0.03125, 0.1659], range ~[-1.1980e-37, 1.1980e-37]: + * |(exp(x)-1-x-x**2/2)/x - p(x)| ~< 2**-122.65 + */ static const double D14 = 1.1470726176204336e-11, /* 0x1.93971dc395d9ep-37 */ D15 = 7.6478532249581686e-13, /* 0x1.ae892e3D16fcep-41 */ @@ -375,6 +205,8 @@ expm1l(long double x) int k, n, n2; uint16_t hx, ix; + DOPRINT_START(&x); + /* Filter out exceptional cases. */ u.e = x; hx = u.xbits.expsign; @@ -382,11 +214,11 @@ expm1l(long double x) if (ix >= BIAS + 7) { /* |x| >= 128 or x is NaN */ if (ix == BIAS + LDBL_MAX_EXP) { if (hx & 0x8000) /* x is -Inf or -NaN */ - return (-1 / x - 1); - return (x + x); /* x is +Inf or +NaN */ + RETURNP(-1 / x - 1); + RETURNP(x + x); /* x is +Inf or +NaN */ } if (x > o_threshold) - return (huge * huge); + RETURNP(huge * huge); /* * expm1l() never underflows, but it must avoid * unrepresentable large negative exponents. We used a @@ -395,7 +227,7 @@ expm1l(long double x) * in the same way as large ones here. */ if (hx & 0x8000) /* x <= -128 */ - return (tiny - 1); /* good for x < -114ln2 - eps */ + RETURN2P(tiny, -1); /* good for x < -114ln2 - eps */ } ENTERI(); @@ -407,7 +239,7 @@ expm1l(long double x) if (x < T3) { if (ix < BIAS - 113) { /* |x| < 0x1p-113 */ /* x (rounded) with inexact if x != 0: */ - RETURNI(x == 0 ? x : + RETURNPI(x == 0 ? x : (0x1p200 * x + fabsl(x)) * 0x1p-200); } q = x * x2 * C3 + x2 * x2 * (C4 + x * (C5 + x * (C6 + @@ -428,9 +260,9 @@ expm1l(long double x) hx2_hi = x_hi * x_hi / 2; hx2_lo = x_lo * (x + x_hi) / 2; if (ix >= BIAS - 7) - RETURNI(hx2_lo + x_lo + q + (hx2_hi + x_hi)); + RETURN2PI(hx2_hi + x_hi, hx2_lo + x_lo + q); else - RETURNI(hx2_lo + q + hx2_hi + x); + RETURN2PI(x, hx2_lo + q + hx2_hi); } /* Reduce x to (k*ln2 + endpoint[n2] + r1 + r2). */ @@ -463,21 +295,21 @@ expm1l(long double x) t = tbl[n2].lo + tbl[n2].hi; if (k == 0) { - t = tbl[n2].lo * (r1 + 1) + t * q + tbl[n2].hi * r1 + - (tbl[n2].hi - 1); + t = SUM2P(tbl[n2].hi - 1, tbl[n2].lo * (r1 + 1) + t * q + + tbl[n2].hi * r1); RETURNI(t); } if (k == -1) { - t = tbl[n2].lo * (r1 + 1) + t * q + tbl[n2].hi * r1 + - (tbl[n2].hi - 2); + t = SUM2P(tbl[n2].hi - 2, tbl[n2].lo * (r1 + 1) + t * q + + tbl[n2].hi * r1); RETURNI(t / 2); } if (k < -7) { - t = tbl[n2].lo + t * (q + r1) + tbl[n2].hi; + t = SUM2P(tbl[n2].hi, tbl[n2].lo + t * (q + r1)); RETURNI(t * twopk - 1); } if (k > 2 * LDBL_MANT_DIG - 1) { - t = tbl[n2].lo + t * (q + r1) + tbl[n2].hi; + t = SUM2P(tbl[n2].hi, tbl[n2].lo + t * (q + r1)); if (k == LDBL_MAX_EXP) RETURNI(t * 2 * 0x1p16383L - 1); RETURNI(t * twopk - 1); @@ -487,8 +319,8 @@ expm1l(long double x) twomk = v.e; if (k > LDBL_MANT_DIG - 1) - t = tbl[n2].lo - twomk + t * (q + r1) + tbl[n2].hi; + t = SUM2P(tbl[n2].hi, tbl[n2].lo - twomk + t * (q + r1)); else - t = tbl[n2].lo + t * (q + r1) + (tbl[n2].hi - twomk); + t = SUM2P(tbl[n2].hi - twomk, tbl[n2].lo + t * (q + r1)); RETURNI(t * twopk); } diff --git a/libm/upstream-freebsd/lib/msun/src/e_cosh.c b/libm/upstream-freebsd/lib/msun/src/e_cosh.c index a36369545..246b5fbec 100644 --- a/libm/upstream-freebsd/lib/msun/src/e_cosh.c +++ b/libm/upstream-freebsd/lib/msun/src/e_cosh.c @@ -35,6 +35,8 @@ __FBSDID("$FreeBSD$"); * only cosh(0)=1 is exact for finite x. */ +#include + #include "math.h" #include "math_private.h" @@ -77,3 +79,7 @@ __ieee754_cosh(double x) /* |x| > overflowthresold, cosh(x) overflow */ return huge*huge; } + +#if (LDBL_MANT_DIG == 53) +__weak_reference(cosh, coshl); +#endif diff --git a/libm/upstream-freebsd/lib/msun/src/e_lgamma_r.c b/libm/upstream-freebsd/lib/msun/src/e_lgamma_r.c index 1cff592c5..7a95ea47f 100644 --- a/libm/upstream-freebsd/lib/msun/src/e_lgamma_r.c +++ b/libm/upstream-freebsd/lib/msun/src/e_lgamma_r.c @@ -86,8 +86,10 @@ __FBSDID("$FreeBSD$"); #include "math.h" #include "math_private.h" +static const volatile double vzero = 0; + static const double -two52= 4.50359962737049600000e+15, /* 0x43300000, 0x00000000 */ +zero= 0.00000000000000000000e+00, half= 5.00000000000000000000e-01, /* 0x3FE00000, 0x00000000 */ one = 1.00000000000000000000e+00, /* 0x3FF00000, 0x00000000 */ pi = 3.14159265358979311600e+00, /* 0x400921FB, 0x54442D18 */ @@ -154,39 +156,35 @@ w4 = -5.95187557450339963135e-04, /* 0xBF4380CB, 0x8C0FE741 */ w5 = 8.36339918996282139126e-04, /* 0x3F4B67BA, 0x4CDAD5D1 */ w6 = -1.63092934096575273989e-03; /* 0xBF5AB89D, 0x0B9E43E4 */ -static const double zero= 0.00000000000000000000e+00; - - static double sin_pi(double x) +/* + * Compute sin(pi*x) without actually doing the pi*x multiplication. + * sin_pi(x) is only called for x < 0 and |x| < 2**(p-1) where p is + * the precision of x. + */ +static double +sin_pi(double x) { + volatile double vz; double y,z; - int n,ix; + int n; + + y = -x; - GET_HIGH_WORD(ix,x); - ix &= 0x7fffffff; + vz = y+0x1p52; /* depend on 0 <= y < 0x1p52 */ + z = vz-0x1p52; /* rint(y) for the above range */ + if (z == y) + return zero; - if(ix<0x3fd00000) return __kernel_sin(pi*x,zero,0); - y = -x; /* x is assume negative */ + vz = y+0x1p50; + GET_LOW_WORD(n,vz); /* bits for rounded y (units 0.25) */ + z = vz-0x1p50; /* y rounded to a multiple of 0.25 */ + if (z > y) { + z -= 0.25; /* adjust to round down */ + n--; + } + n &= 7; /* octant of y mod 2 */ + y = y - z + n * 0.25; /* y mod 2 */ - /* - * argument reduction, make sure inexact flag not raised if input - * is an integer - */ - z = floor(y); - if(z!=y) { /* inexact anyway */ - y *= 0.5; - y = 2.0*(y - floor(y)); /* y = |x| mod 2.0 */ - n = (int) (y*4.0); - } else { - if(ix>=0x43400000) { - y = zero; n = 0; /* y must be even */ - } else { - if(ix<0x43300000) z = y+two52; /* exact */ - GET_LOW_WORD(n,z); - n &= 1; - y = n; - n<<= 2; - } - } switch (n) { case 0: y = __kernel_sin(pi*y,zero,0); break; case 1: @@ -206,7 +204,7 @@ __ieee754_lgamma_r(double x, int *signgamp) { double t,y,z,nadj,p,p1,p2,p3,q,r,w; int32_t hx; - int i,lx,ix; + int i,ix,lx; EXTRACT_WORDS(hx,lx,x); @@ -214,7 +212,7 @@ __ieee754_lgamma_r(double x, int *signgamp) *signgamp = 1; ix = hx&0x7fffffff; if(ix>=0x7ff00000) return x*x; - if((ix|lx)==0) return one/zero; + if((ix|lx)==0) return one/vzero; if(ix<0x3b900000) { /* |x|<2**-70, return -log(|x|) */ if(hx<0) { *signgamp = -1; @@ -223,9 +221,9 @@ __ieee754_lgamma_r(double x, int *signgamp) } if(hx<0) { if(ix>=0x43300000) /* |x|>=2**52, must be -integer */ - return one/zero; + return one/vzero; t = sin_pi(x); - if(t==zero) return one/zero; /* -integer */ + if(t==zero) return one/vzero; /* -integer */ nadj = __ieee754_log(pi/fabs(t*x)); if(t y) { + z -= 0.25F; /* adjust to round down */ + n--; + } + n &= 7; /* octant of y mod 2 */ + y = y - z + n * 0.25F; /* y mod 2 */ - /* - * argument reduction, make sure inexact flag not raised if input - * is an integer - */ - z = floorf(y); - if(z!=y) { /* inexact anyway */ - y *= (float)0.5; - y = (float)2.0*(y - floorf(y)); /* y = |x| mod 2.0 */ - n = (int) (y*(float)4.0); - } else { - if(ix>=0x4b800000) { - y = zero; n = 0; /* y must be even */ - } else { - if(ix<0x4b000000) z = y+two23; /* exact */ - GET_FLOAT_WORD(n,z); - n &= 1; - y = n; - n<<= 2; - } - } switch (n) { case 0: y = __kernel_sindf(pi*y); break; case 1: @@ -147,7 +140,7 @@ __ieee754_lgammaf_r(float x, int *signgamp) *signgamp = 1; ix = hx&0x7fffffff; if(ix>=0x7f800000) return x*x; - if(ix==0) return one/zero; + if(ix==0) return one/vzero; if(ix<0x35000000) { /* |x|<2**-21, return -log(|x|) */ if(hx<0) { *signgamp = -1; @@ -156,9 +149,9 @@ __ieee754_lgammaf_r(float x, int *signgamp) } if(hx<0) { if(ix>=0x4b000000) /* |x|>=2**23, must be -integer */ - return one/zero; + return one/vzero; t = sin_pif(x); - if(t==zero) return one/zero; /* -integer */ + if(t==zero) return one/vzero; /* -integer */ nadj = __ieee754_logf(pi/fabsf(t*x)); if(t 1) ** +INF is +INF * 6. +-(|x| > 1) ** -INF is +0 * 7. +-(|x| < 1) ** +INF is +0 * 8. +-(|x| < 1) ** -INF is +INF - * 9. +-1 ** +-INF is NAN + * 9. +-1 ** +-INF is 1 * 10. +0 ** (+anything except 0, NAN) is +0 * 11. -0 ** (+anything except 0, NAN, odd integer) is +0 * 12. +0 ** (-anything except 0, NAN) is +INF @@ -174,7 +174,7 @@ __ieee754_pow(double x, double y) if(ly==0) { if (iy==0x7ff00000) { /* y is +-inf */ if(((ix-0x3ff00000)|lx)==0) - return one; /* (-1)**+-inf is NaN */ + return one; /* (-1)**+-inf is 1 */ else if (ix >= 0x3ff00000)/* (|x|>1)**+-inf = inf,0 */ return (hy>=0)? y: zero; else /* (|x|<1)**-,+inf = inf,0 */ diff --git a/libm/upstream-freebsd/lib/msun/src/e_sinh.c b/libm/upstream-freebsd/lib/msun/src/e_sinh.c index 17442d0d1..6c01f4a3c 100644 --- a/libm/upstream-freebsd/lib/msun/src/e_sinh.c +++ b/libm/upstream-freebsd/lib/msun/src/e_sinh.c @@ -32,6 +32,8 @@ __FBSDID("$FreeBSD$"); * only sinh(0)=0 is exact for finite x. */ +#include + #include "math.h" #include "math_private.h" @@ -71,3 +73,7 @@ __ieee754_sinh(double x) /* |x| > overflowthresold, sinh(x) overflow */ return x*shuge; } + +#if (LDBL_MANT_DIG == 53) +__weak_reference(sinh, sinhl); +#endif diff --git a/libm/upstream-freebsd/lib/msun/src/imprecise.c b/libm/upstream-freebsd/lib/msun/src/imprecise.c index a7503bf1c..e0ad1031b 100644 --- a/libm/upstream-freebsd/lib/msun/src/imprecise.c +++ b/libm/upstream-freebsd/lib/msun/src/imprecise.c @@ -61,8 +61,6 @@ DECLARE_WEAK(powl); DECLARE_WEAK(f ## l) DECLARE_IMPRECISE(cosh); -DECLARE_IMPRECISE(erfc); -DECLARE_IMPRECISE(erf); DECLARE_IMPRECISE(lgamma); DECLARE_IMPRECISE(sinh); DECLARE_IMPRECISE(tanh); diff --git a/libm/upstream-freebsd/lib/msun/src/s_erf.c b/libm/upstream-freebsd/lib/msun/src/s_erf.c index 0886e5e7d..e1d63bca8 100644 --- a/libm/upstream-freebsd/lib/msun/src/s_erf.c +++ b/libm/upstream-freebsd/lib/msun/src/s_erf.c @@ -111,18 +111,25 @@ __FBSDID("$FreeBSD$"); #include "math.h" #include "math_private.h" +/* XXX Prevent compilers from erroneously constant folding: */ +static const volatile double tiny= 1e-300; + static const double -tiny = 1e-300, -half= 5.00000000000000000000e-01, /* 0x3FE00000, 0x00000000 */ -one = 1.00000000000000000000e+00, /* 0x3FF00000, 0x00000000 */ -two = 2.00000000000000000000e+00, /* 0x40000000, 0x00000000 */ - /* c = (float)0.84506291151 */ +half= 0.5, +one = 1, +two = 2, +/* c = (float)0.84506291151 */ erx = 8.45062911510467529297e-01, /* 0x3FEB0AC1, 0x60000000 */ /* - * Coefficients for approximation to erf on [0,0.84375] + * In the domain [0, 2**-28], only the first term in the power series + * expansion of erf(x) is used. The magnitude of the first neglected + * terms is less than 2**-84. */ efx = 1.28379167095512586316e-01, /* 0x3FC06EBA, 0x8214DB69 */ efx8= 1.02703333676410069053e+00, /* 0x3FF06EBA, 0x8214DB69 */ +/* + * Coefficients for approximation to erf on [0,0.84375] + */ pp0 = 1.28379167095512558561e-01, /* 0x3FC06EBA, 0x8214DB68 */ pp1 = -3.25042107247001499370e-01, /* 0xBFD4CD7D, 0x691CB913 */ pp2 = -2.84817495755985104766e-02, /* 0xBF9D2A51, 0xDBD7194F */ @@ -134,7 +141,7 @@ qq3 = 5.08130628187576562776e-03, /* 0x3F74D022, 0xC4D36B0F */ qq4 = 1.32494738004321644526e-04, /* 0x3F215DC9, 0x221C1A10 */ qq5 = -3.96022827877536812320e-06, /* 0xBED09C43, 0x42A26120 */ /* - * Coefficients for approximation to erf in [0.84375,1.25] + * Coefficients for approximation to erf in [0.84375,1.25] */ pa0 = -2.36211856075265944077e-03, /* 0xBF6359B8, 0xBEF77538 */ pa1 = 4.14856118683748331666e-01, /* 0x3FDA8D00, 0xAD92B34D */ @@ -150,7 +157,7 @@ qa4 = 1.26171219808761642112e-01, /* 0x3FC02660, 0xE763351F */ qa5 = 1.36370839120290507362e-02, /* 0x3F8BEDC2, 0x6B51DD1C */ qa6 = 1.19844998467991074170e-02, /* 0x3F888B54, 0x5735151D */ /* - * Coefficients for approximation to erfc in [1.25,1/0.35] + * Coefficients for approximation to erfc in [1.25,1/0.35] */ ra0 = -9.86494403484714822705e-03, /* 0xBF843412, 0x600D6435 */ ra1 = -6.93858572707181764372e-01, /* 0xBFE63416, 0xE4BA7360 */ @@ -169,7 +176,7 @@ sa6 = 1.08635005541779435134e+02, /* 0x405B28A3, 0xEE48AE2C */ sa7 = 6.57024977031928170135e+00, /* 0x401A47EF, 0x8E484A93 */ sa8 = -6.04244152148580987438e-02, /* 0xBFAEEFF2, 0xEE749A62 */ /* - * Coefficients for approximation to erfc in [1/.35,28] + * Coefficients for approximation to erfc in [1/.35,28] */ rb0 = -9.86494292470009928597e-03, /* 0xBF843412, 0x39E86F4A */ rb1 = -7.99283237680523006574e-01, /* 0xBFE993BA, 0x70C285DE */ @@ -201,7 +208,7 @@ erf(double x) if(ix < 0x3feb0000) { /* |x|<0.84375 */ if(ix < 0x3e300000) { /* |x|<2**-28 */ if (ix < 0x00800000) - return 0.125*(8.0*x+efx8*x); /*avoid underflow */ + return (8*x+efx8*x)/8; /* avoid spurious underflow */ return x + efx*x; } z = x*x; @@ -222,15 +229,12 @@ erf(double x) x = fabs(x); s = one/(x*x); if(ix< 0x4006DB6E) { /* |x| < 1/0.35 */ - R=ra0+s*(ra1+s*(ra2+s*(ra3+s*(ra4+s*( - ra5+s*(ra6+s*ra7)))))); - S=one+s*(sa1+s*(sa2+s*(sa3+s*(sa4+s*( - sa5+s*(sa6+s*(sa7+s*sa8))))))); + R=ra0+s*(ra1+s*(ra2+s*(ra3+s*(ra4+s*(ra5+s*(ra6+s*ra7)))))); + S=one+s*(sa1+s*(sa2+s*(sa3+s*(sa4+s*(sa5+s*(sa6+s*(sa7+ + s*sa8))))))); } else { /* |x| >= 1/0.35 */ - R=rb0+s*(rb1+s*(rb2+s*(rb3+s*(rb4+s*( - rb5+s*rb6))))); - S=one+s*(sb1+s*(sb2+s*(sb3+s*(sb4+s*( - sb5+s*(sb6+s*sb7)))))); + R=rb0+s*(rb1+s*(rb2+s*(rb3+s*(rb4+s*(rb5+s*rb6))))); + S=one+s*(sb1+s*(sb2+s*(sb3+s*(sb4+s*(sb5+s*(sb6+s*sb7)))))); } z = x; SET_LOW_WORD(z,0); @@ -238,6 +242,10 @@ erf(double x) if(hx>=0) return one-r/x; else return r/x-one; } +#if (LDBL_MANT_DIG == 53) +__weak_reference(erf, erfl); +#endif + double erfc(double x) { @@ -279,23 +287,23 @@ erfc(double x) x = fabs(x); s = one/(x*x); if(ix< 0x4006DB6D) { /* |x| < 1/.35 ~ 2.857143*/ - R=ra0+s*(ra1+s*(ra2+s*(ra3+s*(ra4+s*( - ra5+s*(ra6+s*ra7)))))); - S=one+s*(sa1+s*(sa2+s*(sa3+s*(sa4+s*( - sa5+s*(sa6+s*(sa7+s*sa8))))))); + R=ra0+s*(ra1+s*(ra2+s*(ra3+s*(ra4+s*(ra5+s*(ra6+s*ra7)))))); + S=one+s*(sa1+s*(sa2+s*(sa3+s*(sa4+s*(sa5+s*(sa6+s*(sa7+ + s*sa8))))))); } else { /* |x| >= 1/.35 ~ 2.857143 */ if(hx<0&&ix>=0x40180000) return two-tiny;/* x < -6 */ - R=rb0+s*(rb1+s*(rb2+s*(rb3+s*(rb4+s*( - rb5+s*rb6))))); - S=one+s*(sb1+s*(sb2+s*(sb3+s*(sb4+s*( - sb5+s*(sb6+s*sb7)))))); + R=rb0+s*(rb1+s*(rb2+s*(rb3+s*(rb4+s*(rb5+s*rb6))))); + S=one+s*(sb1+s*(sb2+s*(sb3+s*(sb4+s*(sb5+s*(sb6+s*sb7)))))); } z = x; SET_LOW_WORD(z,0); - r = __ieee754_exp(-z*z-0.5625)* - __ieee754_exp((z-x)*(z+x)+R/S); + r = __ieee754_exp(-z*z-0.5625)*__ieee754_exp((z-x)*(z+x)+R/S); if(hx>0) return r/x; else return two-r/x; } else { if(hx>0) return tiny*tiny; else return two-tiny; } } + +#if (LDBL_MANT_DIG == 53) +__weak_reference(erfc, erfcl); +#endif diff --git a/libm/upstream-freebsd/lib/msun/src/s_erff.c b/libm/upstream-freebsd/lib/msun/src/s_erff.c index a3579f1a0..d6cfbd227 100644 --- a/libm/upstream-freebsd/lib/msun/src/s_erff.c +++ b/libm/upstream-freebsd/lib/msun/src/s_erff.c @@ -19,64 +19,63 @@ __FBSDID("$FreeBSD$"); #include "math.h" #include "math_private.h" +/* XXX Prevent compilers from erroneously constant folding: */ +static const volatile float tiny = 1e-30; + static const float -tiny = 1e-30, -half= 5.0000000000e-01, /* 0x3F000000 */ -one = 1.0000000000e+00, /* 0x3F800000 */ -two = 2.0000000000e+00, /* 0x40000000 */ +half= 0.5, +one = 1, +two = 2, +erx = 8.42697144e-01, /* 0x3f57bb00 */ /* - * Coefficients for approximation to erf on [0,0.84375] + * In the domain [0, 2**-14], only the first term in the power series + * expansion of erf(x) is used. The magnitude of the first neglected + * terms is less than 2**-42. */ -efx = 1.2837916613e-01, /* 0x3e0375d4 */ -efx8= 1.0270333290e+00, /* 0x3f8375d4 */ +efx = 1.28379166e-01, /* 0x3e0375d4 */ +efx8= 1.02703333e+00, /* 0x3f8375d4 */ /* - * Domain [0, 0.84375], range ~[-5.4446e-10,5.5197e-10]: - * |(erf(x) - x)/x - p(x)/q(x)| < 2**-31. + * Domain [0, 0.84375], range ~[-5.4419e-10, 5.5179e-10]: + * |(erf(x) - x)/x - pp(x)/qq(x)| < 2**-31 */ -pp0 = 1.28379166e-01F, /* 0x1.06eba8p-3 */ -pp1 = -3.36030394e-01F, /* -0x1.58185ap-2 */ -pp2 = -1.86260219e-03F, /* -0x1.e8451ep-10 */ -qq1 = 3.12324286e-01F, /* 0x1.3fd1f0p-2 */ -qq2 = 2.16070302e-02F, /* 0x1.620274p-6 */ -qq3 = -1.98859419e-03F, /* -0x1.04a626p-9 */ +pp0 = 1.28379166e-01, /* 0x3e0375d4 */ +pp1 = -3.36030394e-01, /* 0xbeac0c2d */ +pp2 = -1.86261395e-03, /* 0xbaf422f4 */ +qq1 = 3.12324315e-01, /* 0x3e9fe8f9 */ +qq2 = 2.16070414e-02, /* 0x3cb10140 */ +qq3 = -1.98859372e-03, /* 0xbb025311 */ /* - * Domain [0.84375, 1.25], range ~[-1.953e-11,1.940e-11]: - * |(erf(x) - erx) - p(x)/q(x)| < 2**-36. + * Domain [0.84375, 1.25], range ~[-1.023e-9, 1.023e-9]: + * |(erf(x) - erx) - pa(x)/qa(x)| < 2**-31 */ -erx = 8.42697144e-01F, /* 0x1.af7600p-1. erf(1) rounded to 16 bits. */ -pa0 = 3.64939137e-06F, /* 0x1.e9d022p-19 */ -pa1 = 4.15109694e-01F, /* 0x1.a91284p-2 */ -pa2 = -1.65179938e-01F, /* -0x1.5249dcp-3 */ -pa3 = 1.10914491e-01F, /* 0x1.c64e46p-4 */ -qa1 = 6.02074385e-01F, /* 0x1.344318p-1 */ -qa2 = 5.35934687e-01F, /* 0x1.126608p-1 */ -qa3 = 1.68576106e-01F, /* 0x1.593e6ep-3 */ -qa4 = 5.62181212e-02F, /* 0x1.cc89f2p-5 */ +pa0 = 3.65041046e-06, /* 0x3674f993 */ +pa1 = 4.15109307e-01, /* 0x3ed48935 */ +pa2 = -2.09395722e-01, /* 0xbe566bd5 */ +pa3 = 8.67677554e-02, /* 0x3db1b34b */ +qa1 = 4.95560974e-01, /* 0x3efdba2b */ +qa2 = 3.71248513e-01, /* 0x3ebe1449 */ +qa3 = 3.92478965e-02, /* 0x3d20c267 */ /* - * Domain [1.25,1/0.35], range ~[-7.043e-10,7.457e-10]: - * |log(x*erfc(x)) + x**2 + 0.5625 - r(x)/s(x)| < 2**-30 + * Domain [1.25,1/0.35], range ~[-4.821e-9, 4.927e-9]: + * |log(x*erfc(x)) + x**2 + 0.5625 - ra(x)/sa(x)| < 2**-28 */ -ra0 = -9.87132732e-03F, /* -0x1.4376b2p-7 */ -ra1 = -5.53605914e-01F, /* -0x1.1b723cp-1 */ -ra2 = -2.17589188e+00F, /* -0x1.1683a0p+1 */ -ra3 = -1.43268085e+00F, /* -0x1.6ec42cp+0 */ -sa1 = 5.45995426e+00F, /* 0x1.5d6fe4p+2 */ -sa2 = 6.69798088e+00F, /* 0x1.acabb8p+2 */ -sa3 = 1.43113089e+00F, /* 0x1.6e5e98p+0 */ -sa4 = -5.77397496e-02F, /* -0x1.d90108p-5 */ +ra0 = -9.88156721e-03, /* 0xbc21e64c */ +ra1 = -5.43658376e-01, /* 0xbf0b2d32 */ +ra2 = -1.66828310e+00, /* 0xbfd58a4d */ +ra3 = -6.91554189e-01, /* 0xbf3109b2 */ +sa1 = 4.48581553e+00, /* 0x408f8bcd */ +sa2 = 4.10799170e+00, /* 0x408374ab */ +sa3 = 5.53855181e-01, /* 0x3f0dc974 */ /* - * Domain [1/0.35, 11], range ~[-2.264e-13,2.336e-13]: - * |log(x*erfc(x)) + x**2 + 0.5625 - r(x)/s(x)| < 2**-42 + * Domain [2.85715, 11], range ~[-1.484e-9, 1.505e-9]: + * |log(x*erfc(x)) + x**2 + 0.5625 - rb(x)/sb(x)| < 2**-30 */ -rb0 = -9.86494310e-03F, /* -0x1.434124p-7 */ -rb1 = -6.25171244e-01F, /* -0x1.401672p-1 */ -rb2 = -6.16498327e+00F, /* -0x1.8a8f16p+2 */ -rb3 = -1.66696873e+01F, /* -0x1.0ab70ap+4 */ -rb4 = -9.53764343e+00F, /* -0x1.313460p+3 */ -sb1 = 1.26884899e+01F, /* 0x1.96081cp+3 */ -sb2 = 4.51839523e+01F, /* 0x1.6978bcp+5 */ -sb3 = 4.72810211e+01F, /* 0x1.7a3f88p+5 */ -sb4 = 8.93033314e+00F; /* 0x1.1dc54ap+3 */ +rb0 = -9.86496918e-03, /* 0xbc21a0ae */ +rb1 = -5.48049808e-01, /* 0xbf0c4cfe */ +rb2 = -1.84115684e+00, /* 0xbfebab07 */ +sb1 = 4.87132740e+00, /* 0x409be1ea */ +sb2 = 3.04982710e+00, /* 0x4043305e */ +sb3 = -7.61900663e-01; /* 0xbf430bec */ float erff(float x) @@ -85,9 +84,9 @@ erff(float x) float R,S,P,Q,s,y,z,r; GET_FLOAT_WORD(hx,x); ix = hx&0x7fffffff; - if(ix>=0x7f800000) { /* erf(nan)=nan */ + if(ix>=0x7f800000) { /* erff(nan)=nan */ i = ((u_int32_t)hx>>31)<<1; - return (float)(1-i)+one/x; /* erf(+-inf)=+-1 */ + return (float)(1-i)+one/x; /* erff(+-inf)=+-1 */ } if(ix < 0x3f580000) { /* |x|<0.84375 */ @@ -105,7 +104,7 @@ erff(float x) if(ix < 0x3fa00000) { /* 0.84375 <= |x| < 1.25 */ s = fabsf(x)-one; P = pa0+s*(pa1+s*(pa2+s*pa3)); - Q = one+s*(qa1+s*(qa2+s*(qa3+s*qa4))); + Q = one+s*(qa1+s*(qa2+s*qa3)); if(hx>=0) return erx + P/Q; else return -erx - P/Q; } if (ix >= 0x40800000) { /* inf>|x|>=4 */ @@ -113,12 +112,12 @@ erff(float x) } x = fabsf(x); s = one/(x*x); - if(ix< 0x4036DB6E) { /* |x| < 1/0.35 */ + if(ix< 0x4036db8c) { /* |x| < 2.85715 ~ 1/0.35 */ R=ra0+s*(ra1+s*(ra2+s*ra3)); - S=one+s*(sa1+s*(sa2+s*(sa3+s*sa4))); - } else { /* |x| >= 1/0.35 */ - R=rb0+s*(rb1+s*(rb2+s*(rb3+s*rb4))); - S=one+s*(sb1+s*(sb2+s*(sb3+s*sb4))); + S=one+s*(sa1+s*(sa2+s*sa3)); + } else { /* |x| >= 2.85715 ~ 1/0.35 */ + R=rb0+s*(rb1+s*rb2); + S=one+s*(sb1+s*(sb2+s*sb3)); } SET_FLOAT_WORD(z,hx&0xffffe000); r = expf(-z*z-0.5625F)*expf((z-x)*(z+x)+R/S); @@ -132,8 +131,8 @@ erfcf(float x) float R,S,P,Q,s,y,z,r; GET_FLOAT_WORD(hx,x); ix = hx&0x7fffffff; - if(ix>=0x7f800000) { /* erfc(nan)=nan */ - /* erfc(+-inf)=0,2 */ + if(ix>=0x7f800000) { /* erfcf(nan)=nan */ + /* erfcf(+-inf)=0,2 */ return (float)(((u_int32_t)hx>>31)<<1)+one/x; } @@ -155,7 +154,7 @@ erfcf(float x) if(ix < 0x3fa00000) { /* 0.84375 <= |x| < 1.25 */ s = fabsf(x)-one; P = pa0+s*(pa1+s*(pa2+s*pa3)); - Q = one+s*(qa1+s*(qa2+s*(qa3+s*qa4))); + Q = one+s*(qa1+s*(qa2+s*qa3)); if(hx>=0) { z = one-erx; return z - P/Q; } else { @@ -165,13 +164,13 @@ erfcf(float x) if (ix < 0x41300000) { /* |x|<11 */ x = fabsf(x); s = one/(x*x); - if(ix< 0x4036DB6D) { /* |x| < 1/.35 ~ 2.857143*/ - R=ra0+s*(ra1+s*(ra2+s*ra3)); - S=one+s*(sa1+s*(sa2+s*(sa3+s*sa4))); - } else { /* |x| >= 1/.35 ~ 2.857143 */ + if(ix< 0x4036db8c) { /* |x| < 2.85715 ~ 1/.35 */ + R=ra0+s*(ra1+s*(ra2+s*ra3)); + S=one+s*(sa1+s*(sa2+s*sa3)); + } else { /* |x| >= 2.85715 ~ 1/.35 */ if(hx<0&&ix>=0x40a00000) return two-tiny;/* x < -5 */ - R=rb0+s*(rb1+s*(rb2+s*(rb3+s*rb4))); - S=one+s*(sb1+s*(sb2+s*(sb3+s*sb4))); + R=rb0+s*(rb1+s*rb2); + S=one+s*(sb1+s*(sb2+s*sb3)); } SET_FLOAT_WORD(z,hx&0xffffe000); r = expf(-z*z-0.5625F)*expf((z-x)*(z+x)+R/S); diff --git a/libm/upstream-freebsd/lib/msun/src/s_rintl.c b/libm/upstream-freebsd/lib/msun/src/s_rintl.c index e55b40e5b..b43df89f5 100644 --- a/libm/upstream-freebsd/lib/msun/src/s_rintl.c +++ b/libm/upstream-freebsd/lib/msun/src/s_rintl.c @@ -29,7 +29,6 @@ __FBSDID("$FreeBSD$"); #include #include -#include #include "fpmath.h" diff --git a/libm/upstream-freebsd/lib/msun/src/s_tanh.c b/libm/upstream-freebsd/lib/msun/src/s_tanh.c index 96e35656b..6d26c695d 100644 --- a/libm/upstream-freebsd/lib/msun/src/s_tanh.c +++ b/libm/upstream-freebsd/lib/msun/src/s_tanh.c @@ -37,10 +37,13 @@ __FBSDID("$FreeBSD$"); * only tanh(0)=0 is exact for finite argument. */ +#include + #include "math.h" #include "math_private.h" -static const double one = 1.0, two = 2.0, tiny = 1.0e-300, huge = 1.0e300; +static const volatile double tiny = 1.0e-300; +static const double one = 1.0, two = 2.0, huge = 1.0e300; double tanh(double x) @@ -75,3 +78,7 @@ tanh(double x) } return (jx>=0)? z: -z; } + +#if (LDBL_MANT_DIG == 53) +__weak_reference(tanh, tanhl); +#endif diff --git a/libm/upstream-freebsd/lib/msun/src/s_tanhf.c b/libm/upstream-freebsd/lib/msun/src/s_tanhf.c index 04f09c686..f537be4fd 100644 --- a/libm/upstream-freebsd/lib/msun/src/s_tanhf.c +++ b/libm/upstream-freebsd/lib/msun/src/s_tanhf.c @@ -19,7 +19,9 @@ __FBSDID("$FreeBSD$"); #include "math.h" #include "math_private.h" -static const float one=1.0, two=2.0, tiny = 1.0e-30, huge = 1.0e30; +static const volatile float tiny = 1.0e-30; +static const float one=1.0, two=2.0, huge = 1.0e30; + float tanhf(float x) { From bee6fbf31dccee966a1d4ff1a5e0ed888a4e5dfc Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Fri, 12 Sep 2014 16:09:40 -0700 Subject: [PATCH 71/89] Get new coshl, sinhl, and tanhl implementations from upstream. Change-Id: I92f9cd53d12efb2e0f4f8a9590b9fe42398233f6 --- libm/Android.mk | 3 + libm/upstream-freebsd/lib/msun/src/e_coshl.c | 130 +++++++++++++ libm/upstream-freebsd/lib/msun/src/e_sinhl.c | 131 +++++++++++++ .../upstream-freebsd/lib/msun/src/imprecise.c | 3 - libm/upstream-freebsd/lib/msun/src/s_tanhl.c | 172 ++++++++++++++++++ 5 files changed, 436 insertions(+), 3 deletions(-) create mode 100644 libm/upstream-freebsd/lib/msun/src/e_coshl.c create mode 100644 libm/upstream-freebsd/lib/msun/src/e_sinhl.c create mode 100644 libm/upstream-freebsd/lib/msun/src/s_tanhl.c diff --git a/libm/Android.mk b/libm/Android.mk index 379aea70a..2e32e3ea3 100644 --- a/libm/Android.mk +++ b/libm/Android.mk @@ -190,6 +190,7 @@ libm_ld_src_files = \ upstream-freebsd/lib/msun/src/s_cbrtl.c \ upstream-freebsd/lib/msun/src/s_ceill.c \ upstream-freebsd/lib/msun/src/s_copysignl.c \ + upstream-freebsd/lib/msun/src/e_coshl.c \ upstream-freebsd/lib/msun/src/s_cosl.c \ upstream-freebsd/lib/msun/src/s_fabsl.c \ upstream-freebsd/lib/msun/src/s_floorl.c \ @@ -211,7 +212,9 @@ libm_ld_src_files = \ upstream-freebsd/lib/msun/src/s_rintl.c \ upstream-freebsd/lib/msun/src/s_roundl.c \ upstream-freebsd/lib/msun/src/s_scalbnl.c \ + upstream-freebsd/lib/msun/src/e_sinhl.c \ upstream-freebsd/lib/msun/src/s_sinl.c \ + upstream-freebsd/lib/msun/src/s_tanhl.c \ upstream-freebsd/lib/msun/src/s_tanl.c \ upstream-freebsd/lib/msun/src/s_truncl.c \ diff --git a/libm/upstream-freebsd/lib/msun/src/e_coshl.c b/libm/upstream-freebsd/lib/msun/src/e_coshl.c new file mode 100644 index 000000000..0a21277fc --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/src/e_coshl.c @@ -0,0 +1,130 @@ +/* from: FreeBSD: head/lib/msun/src/e_coshl.c XXX */ + +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +#include +__FBSDID("$FreeBSD$"); + +/* + * See e_cosh.c for complete comments. + * + * Converted to long double by Bruce D. Evans. + */ + +#include +#ifdef __i386__ +#include +#endif + +#include "fpmath.h" +#include "math.h" +#include "math_private.h" +#include "k_expl.h" + +#if LDBL_MAX_EXP != 0x4000 +/* We also require the usual expsign encoding. */ +#error "Unsupported long double format" +#endif + +#define BIAS (LDBL_MAX_EXP - 1) + +static const volatile long double huge = 0x1p10000L, tiny = 0x1p-10000L; +#if LDBL_MANT_DIG == 64 +/* + * Domain [-1, 1], range ~[-1.8211e-21, 1.8211e-21]: + * |cosh(x) - c(x)| < 2**-68.8 + */ +static const union IEEEl2bits +C4u = LD80C(0xaaaaaaaaaaaaac78, -5, 4.16666666666666682297e-2L); +#define C4 C4u.e +static const double +C2 = 0.5, +C6 = 1.3888888888888616e-3, /* 0x16c16c16c16b99.0p-62 */ +C8 = 2.4801587301767953e-5, /* 0x1a01a01a027061.0p-68 */ +C10 = 2.7557319163300398e-7, /* 0x127e4fb6c9b55f.0p-74 */ +C12 = 2.0876768371393075e-9, /* 0x11eed99406a3f4.0p-81 */ +C14 = 1.1469537039374480e-11, /* 0x1938c67cd18c48.0p-89 */ +C16 = 4.8473490896852041e-14; /* 0x1b49c429701e45.0p-97 */ +#elif LDBL_MANT_DIG == 113 +/* + * Domain [-1, 1], range ~[-2.3194e-37, 2.3194e-37]: + * |cosh(x) - c(x)| < 2**-121.69 + */ +static const long double +C4 = 4.16666666666666666666666666666666225e-2L, /* 0x1555555555555555555555555554e.0p-117L */ +C6 = 1.38888888888888888888888888889434831e-3L, /* 0x16c16c16c16c16c16c16c16c1dd7a.0p-122L */ +C8 = 2.48015873015873015873015871870962089e-5L, /* 0x1a01a01a01a01a01a01a017af2756.0p-128L */ +C10 = 2.75573192239858906525574318600800201e-7L, /* 0x127e4fb7789f5c72ef01c8a040640.0p-134L */ +C12 = 2.08767569878680989791444691755468269e-9L, /* 0x11eed8eff8d897b543d0679607399.0p-141L */ +C14= 1.14707455977297247387801189650495351e-11L, /* 0x193974a8c07c9d24ae169a7fa9b54.0p-149L */ +C16 = 4.77947733238737883626416876486279985e-14L; /* 0x1ae7f3e733b814d4e1b90f5727fe4.0p-157L */ +static const double +C2 = 0.5, +C18 = 1.5619206968597871e-16, /* 0x16827863b9900b.0p-105 */ +C20 = 4.1103176218528049e-19, /* 0x1e542ba3d3c269.0p-114 */ +C22 = 8.8967926401641701e-22, /* 0x10ce399542a014.0p-122 */ +C24 = 1.6116681626523904e-24, /* 0x1f2c981d1f0cb7.0p-132 */ +C26 = 2.5022374732804632e-27; /* 0x18c7ecf8b2c4a0.0p-141 */ +#else +#error "Unsupported long double format" +#endif /* LDBL_MANT_DIG == 64 */ + +/* log(2**16385 - 0.5) rounded up: */ +static const float +o_threshold = 1.13572168e4; /* 0xb174de.0p-10 */ + +long double +coshl(long double x) +{ + long double hi,lo,x2,x4; + double dx2; + uint16_t ix; + + GET_LDBL_EXPSIGN(ix,x); + ix &= 0x7fff; + + /* x is INF or NaN */ + if(ix>=0x7fff) return x*x; + + ENTERI(); + + /* |x| < 1, return 1 or c(x) */ + if(ix<0x3fff) { + if (ix o_threshold, cosh(x) overflow */ + RETURNI(huge*huge); +} diff --git a/libm/upstream-freebsd/lib/msun/src/e_sinhl.c b/libm/upstream-freebsd/lib/msun/src/e_sinhl.c new file mode 100644 index 000000000..ce7e3333d --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/src/e_sinhl.c @@ -0,0 +1,131 @@ +/* from: FreeBSD: head/lib/msun/src/e_sinhl.c XXX */ + +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +#include +__FBSDID("$FreeBSD$"); + +/* + * See e_sinh.c for complete comments. + * + * Converted to long double by Bruce D. Evans. + */ + +#include +#ifdef __i386__ +#include +#endif + +#include "fpmath.h" +#include "math.h" +#include "math_private.h" +#include "k_expl.h" + +#if LDBL_MAX_EXP != 0x4000 +/* We also require the usual expsign encoding. */ +#error "Unsupported long double format" +#endif + +#define BIAS (LDBL_MAX_EXP - 1) + +static const long double shuge = 0x1p16383L; +#if LDBL_MANT_DIG == 64 +/* + * Domain [-1, 1], range ~[-6.6749e-22, 6.6749e-22]: + * |sinh(x)/x - s(x)| < 2**-70.3 + */ +static const union IEEEl2bits +S3u = LD80C(0xaaaaaaaaaaaaaaaa, -3, 1.66666666666666666658e-1L); +#define S3 S3u.e +static const double +S5 = 8.3333333333333332e-3, /* 0x11111111111111.0p-59 */ +S7 = 1.9841269841270074e-4, /* 0x1a01a01a01a070.0p-65 */ +S9 = 2.7557319223873889e-6, /* 0x171de3a5565fe6.0p-71 */ +S11 = 2.5052108406704084e-8, /* 0x1ae6456857530f.0p-78 */ +S13 = 1.6059042748655297e-10, /* 0x161245fa910697.0p-85 */ +S15 = 7.6470006914396920e-13, /* 0x1ae7ce4eff2792.0p-93 */ +S17 = 2.8346142308424267e-15; /* 0x19882ce789ffc6.0p-101 */ +#elif LDBL_MANT_DIG == 113 +/* + * Domain [-1, 1], range ~[-2.9673e-36, 2.9673e-36]: + * |sinh(x)/x - s(x)| < 2**-118.0 + */ +static const long double +S3 = 1.66666666666666666666666666666666033e-1L, /* 0x1555555555555555555555555553b.0p-115L */ +S5 = 8.33333333333333333333333333337643193e-3L, /* 0x111111111111111111111111180f5.0p-119L */ +S7 = 1.98412698412698412698412697391263199e-4L, /* 0x1a01a01a01a01a01a01a0176aad11.0p-125L */ +S9 = 2.75573192239858906525574406205464218e-6L, /* 0x171de3a556c7338faac243aaa9592.0p-131L */ +S11 = 2.50521083854417187749675637460977997e-8L, /* 0x1ae64567f544e38fe59b3380d7413.0p-138L */ +S13 = 1.60590438368216146368737762431552702e-10L, /* 0x16124613a86d098059c7620850fc2.0p-145L */ +S15 = 7.64716373181980539786802470969096440e-13L, /* 0x1ae7f3e733b814193af09ce723043.0p-153L */ +S17 = 2.81145725434775409870584280722701574e-15L; /* 0x1952c77030c36898c3fd0b6dfc562.0p-161L */ +static const double +S19= 8.2206352435411005e-18, /* 0x12f49b4662b86d.0p-109 */ +S21= 1.9572943931418891e-20, /* 0x171b8f2fab9628.0p-118 */ +S23 = 3.8679983530666939e-23, /* 0x17617002b73afc.0p-127 */ +S25 = 6.5067867911512749e-26; /* 0x1423352626048a.0p-136 */ +#else +#error "Unsupported long double format" +#endif /* LDBL_MANT_DIG == 64 */ + +/* log(2**16385 - 0.5) rounded up: */ +static const float +o_threshold = 1.13572168e4; /* 0xb174de.0p-10 */ + +long double +sinhl(long double x) +{ + long double hi,lo,x2,x4; + double dx2,s; + int16_t ix,jx; + + GET_LDBL_EXPSIGN(jx,x); + ix = jx&0x7fff; + + /* x is INF or NaN */ + if(ix>=0x7fff) return x+x; + + ENTERI(); + + s = 1; + if (jx<0) s = -1; + + /* |x| < 64, return x, s(x), or accurate s*(exp(|x|)/2-1/exp(|x|)/2) */ + if (ix<0x4005) { /* |x|<64 */ + if (ix1) RETURNI(x); /* sinh(tiny) = tiny with inexact */ + if (ix<0x3fff) { /* |x|<1 */ + x2 = x*x; +#if LDBL_MANT_DIG == 64 + x4 = x2*x2; + RETURNI(((S17*x2 + S15)*x4 + (S13*x2 + S11))*(x2*x*x4*x4) + + ((S9*x2 + S7)*x2 + S5)*(x2*x*x2) + S3*(x2*x) + x); +#elif LDBL_MANT_DIG == 113 + dx2 = x2; + RETURNI(((((((((((S25*dx2 + S23)*dx2 + + S21)*x2 + S19)*x2 + + S17)*x2 + S15)*x2 + S13)*x2 + S11)*x2 + S9)*x2 + S7)*x2 + + S5)* (x2*x*x2) + + S3*(x2*x) + x); +#endif + } + k_hexpl(fabsl(x), &hi, &lo); + RETURNI(s*(lo - 0.25/(hi + lo) + hi)); + } + + /* |x| in [64, o_threshold], return correctly-overflowing s*exp(|x|)/2 */ + if (fabsl(x) <= o_threshold) + RETURNI(s*hexpl(fabsl(x))); + + /* |x| > o_threshold, sinh(x) overflow */ + return x*shuge; +} diff --git a/libm/upstream-freebsd/lib/msun/src/imprecise.c b/libm/upstream-freebsd/lib/msun/src/imprecise.c index e0ad1031b..92fb2d06e 100644 --- a/libm/upstream-freebsd/lib/msun/src/imprecise.c +++ b/libm/upstream-freebsd/lib/msun/src/imprecise.c @@ -60,8 +60,5 @@ DECLARE_WEAK(powl); long double imprecise_ ## f ## l(long double v) { return f(v); }\ DECLARE_WEAK(f ## l) -DECLARE_IMPRECISE(cosh); DECLARE_IMPRECISE(lgamma); -DECLARE_IMPRECISE(sinh); -DECLARE_IMPRECISE(tanh); DECLARE_IMPRECISE(tgamma); diff --git a/libm/upstream-freebsd/lib/msun/src/s_tanhl.c b/libm/upstream-freebsd/lib/msun/src/s_tanhl.c new file mode 100644 index 000000000..886158b78 --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/src/s_tanhl.c @@ -0,0 +1,172 @@ +/* from: FreeBSD: head/lib/msun/src/s_tanhl.c XXX */ + +/* @(#)s_tanh.c 5.1 93/09/24 */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +#include +__FBSDID("$FreeBSD$"); + +/* + * See s_tanh.c for complete comments. + * + * Converted to long double by Bruce D. Evans. + */ + +#include +#ifdef __i386__ +#include +#endif + +#include "math.h" +#include "math_private.h" +#include "fpmath.h" +#include "k_expl.h" + +#if LDBL_MAX_EXP != 0x4000 +/* We also require the usual expsign encoding. */ +#error "Unsupported long double format" +#endif + +#define BIAS (LDBL_MAX_EXP - 1) + +static const volatile double tiny = 1.0e-300; +static const double one = 1.0; +#if LDBL_MANT_DIG == 64 +/* + * Domain [-0.25, 0.25], range ~[-1.6304e-22, 1.6304e-22]: + * |tanh(x)/x - t(x)| < 2**-72.3 + */ +static const union IEEEl2bits +T3u = LD80C(0xaaaaaaaaaaaaaa9f, -2, -3.33333333333333333017e-1L); +#define T3 T3u.e +static const double +T5 = 1.3333333333333314e-1, /* 0x1111111111110a.0p-55 */ +T7 = -5.3968253968210485e-2, /* -0x1ba1ba1ba1a1a1.0p-57 */ +T9 = 2.1869488531393817e-2, /* 0x1664f488172022.0p-58 */ +T11 = -8.8632352345964591e-3, /* -0x1226e34bc138d5.0p-59 */ +T13 = 3.5921169709993771e-3, /* 0x1d6d371d3e400f.0p-61 */ +T15 = -1.4555786415756001e-3, /* -0x17d923aa63814d.0p-62 */ +T17 = 5.8645267876296793e-4, /* 0x13378589b85aa7.0p-63 */ +T19 = -2.1121033571392224e-4; /* -0x1baf0af80c4090.0p-65 */ +#elif LDBL_MANT_DIG == 113 +/* + * Domain [-0.25, 0.25], range ~[-2.4211e-37, 2.4211e-37]: + * |tanh(x)/x - t(x)| < 2**121.6 + */ +static const long double +T3 = -3.33333333333333333333333333333332980e-1L, /* -0x1555555555555555555555555554e.0p-114L */ +T5 = 1.33333333333333333333333333332707260e-1L, /* 0x1111111111111111111111110ab7b.0p-115L */ +T7 = -5.39682539682539682539682535723482314e-2L, /* -0x1ba1ba1ba1ba1ba1ba1ba17b5fc98.0p-117L */ +T9 = 2.18694885361552028218693591149061717e-2L, /* 0x1664f4882c10f9f32d6b1a12a25e5.0p-118L */ +T11 = -8.86323552990219656883762347736381851e-3L, /* -0x1226e355e6c23c8f5a5a0f386cb4d.0p-119L */ +T13 = 3.59212803657248101358314398220822722e-3L, /* 0x1d6d3d0e157ddfb403ad3637442c6.0p-121L */ +T15 = -1.45583438705131796512568010348874662e-3L; /* -0x17da36452b75e150c44cc34253b34.0p-122L */ +static const double +T17 = 5.9002744094556621e-4, /* 0x1355824803668e.0p-63 */ +T19 = -2.3912911424260516e-4, /* -0x1f57d7734c8dde.0p-65 */ +T21 = 9.6915379535512898e-5, /* 0x1967e18ad6a6ca.0p-66 */ +T23 = -3.9278322983156353e-5, /* -0x1497d8e6b75729.0p-67 */ +T25 = 1.5918887220143869e-5, /* 0x10b1319998cafa.0p-68 */ +T27 = -6.4514295231630956e-6, /* -0x1b0f2b71b218eb.0p-70 */ +T29 = 2.6120754043964365e-6, /* 0x15e963a3cf3a39.0p-71 */ +T31 = -1.0407567231003314e-6, /* -0x1176041e656869.0p-72 */ +T33 = 3.4744117554063574e-7; /* 0x1750fe732cab9c.0p-74 */ +#endif /* LDBL_MANT_DIG == 64 */ + +static inline long double +divl(long double a, long double b, long double c, long double d, + long double e, long double f) +{ + long double inv, r; + float fr, fw; + + _2sumF(a, c); + b = b + c; + _2sumF(d, f); + e = e + f; + + inv = 1 / (d + e); + + r = (a + b) * inv; + fr = r; + r = fr; + + fw = d + e; + e = d - fw + e; + d = fw; + + r = r + (a - d * r + b - e * r) * inv; + + return r; +} + +long double +tanhl(long double x) +{ + long double hi,lo,s,x2,x4,z; + double dx2; + int16_t jx,ix; + + GET_LDBL_EXPSIGN(jx,x); + ix = jx&0x7fff; + + /* x is INF or NaN */ + if(ix>=0x7fff) { + if (jx>=0) return one/x+one; /* tanh(+-inf)=+-1 */ + else return one/x-one; /* tanh(NaN) = NaN */ + } + + ENTERI(); + + /* |x| < 40 */ + if (ix < 0x4004 || fabsl(x) < 40) { /* |x|<40 */ + if (__predict_false(ix= 40, return +-1 */ + } else { + z = one - tiny; /* raise inexact flag */ + } + s = 1; + if (jx<0) s = -1; + RETURNI(s*z); +} From 8c0b20fc497398ca6673e243dbf7d3f5c1fdefcf Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Thu, 6 Nov 2014 11:16:55 -0800 Subject: [PATCH 72/89] Fix our support. We build libm with -fvisibility=hidden, so we weren't exporting any of the functions. We also weren't building many of the functions anyway. We were also missing the complex inverse trigonometric functions. And because we didn't even have perfunctory "call each function once" tests, we didn't notice that we weren't exporting any symbols, so this patch adds at least that level of testing. Change-Id: Ibcf2843f507126c51d134cc5fc8d67747e033a0d --- libm/Android.mk | 19 +- libm/include/complex.h | 26 + libm/upstream-freebsd/lib/msun/src/catrig.c | 639 +++++++++++++++++++ libm/upstream-freebsd/lib/msun/src/catrigf.c | 393 ++++++++++++ tests/Android.mk | 1 + tests/complex_test.cpp | 260 ++++++++ 6 files changed, 1333 insertions(+), 5 deletions(-) create mode 100644 libm/upstream-freebsd/lib/msun/src/catrig.c create mode 100644 libm/upstream-freebsd/lib/msun/src/catrigf.c create mode 100644 tests/complex_test.cpp diff --git a/libm/Android.mk b/libm/Android.mk index 2e32e3ea3..27967c5e4 100644 --- a/libm/Android.mk +++ b/libm/Android.mk @@ -16,6 +16,8 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/bsdsrc/b_exp.c \ upstream-freebsd/lib/msun/bsdsrc/b_log.c \ upstream-freebsd/lib/msun/bsdsrc/b_tgamma.c \ + upstream-freebsd/lib/msun/src/catrig.c \ + upstream-freebsd/lib/msun/src/catrigf.c \ upstream-freebsd/lib/msun/src/e_acos.c \ upstream-freebsd/lib/msun/src/e_acosf.c \ upstream-freebsd/lib/msun/src/e_acosh.c \ @@ -80,6 +82,7 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_atanf.c \ upstream-freebsd/lib/msun/src/s_carg.c \ upstream-freebsd/lib/msun/src/s_cargf.c \ + upstream-freebsd/lib/msun/src/s_cargl.c \ upstream-freebsd/lib/msun/src/s_cbrt.c \ upstream-freebsd/lib/msun/src/s_cbrtf.c \ upstream-freebsd/lib/msun/src/s_ccosh.c \ @@ -90,19 +93,24 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_cexpf.c \ upstream-freebsd/lib/msun/src/s_cimag.c \ upstream-freebsd/lib/msun/src/s_cimagf.c \ + upstream-freebsd/lib/msun/src/s_cimagl.c \ upstream-freebsd/lib/msun/src/s_conj.c \ upstream-freebsd/lib/msun/src/s_conjf.c \ + upstream-freebsd/lib/msun/src/s_conjl.c \ upstream-freebsd/lib/msun/src/s_copysign.c \ upstream-freebsd/lib/msun/src/s_copysignf.c \ upstream-freebsd/lib/msun/src/s_cosf.c \ upstream-freebsd/lib/msun/src/s_cproj.c \ upstream-freebsd/lib/msun/src/s_cprojf.c \ + upstream-freebsd/lib/msun/src/s_cprojl.c \ upstream-freebsd/lib/msun/src/s_creal.c \ upstream-freebsd/lib/msun/src/s_crealf.c \ + upstream-freebsd/lib/msun/src/s_creall.c \ upstream-freebsd/lib/msun/src/s_csinh.c \ upstream-freebsd/lib/msun/src/s_csinhf.c \ upstream-freebsd/lib/msun/src/s_csqrt.c \ upstream-freebsd/lib/msun/src/s_csqrtf.c \ + upstream-freebsd/lib/msun/src/s_csqrtl.c \ upstream-freebsd/lib/msun/src/s_ctanh.c \ upstream-freebsd/lib/msun/src/s_ctanhf.c \ upstream-freebsd/lib/msun/src/s_erf.c \ @@ -168,6 +176,7 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_truncf.c \ upstream-freebsd/lib/msun/src/w_cabs.c \ upstream-freebsd/lib/msun/src/w_cabsf.c \ + upstream-freebsd/lib/msun/src/w_cabsl.c \ upstream-freebsd/lib/msun/src/w_drem.c \ upstream-freebsd/lib/msun/src/w_dremf.c @@ -175,7 +184,7 @@ libm_common_src_files += \ fake_long_double.c \ signbit.c \ -libm_ld_src_files = \ +libm_ld128_src_files = \ upstream-freebsd/lib/msun/src/e_acosl.c \ upstream-freebsd/lib/msun/src/e_acoshl.c \ upstream-freebsd/lib/msun/src/e_asinl.c \ @@ -218,7 +227,7 @@ libm_ld_src_files = \ upstream-freebsd/lib/msun/src/s_tanl.c \ upstream-freebsd/lib/msun/src/s_truncl.c \ -libm_ld_src_files += \ +libm_ld128_src_files += \ upstream-freebsd/lib/msun/ld128/invtrig.c \ upstream-freebsd/lib/msun/ld128/k_cosl.c \ upstream-freebsd/lib/msun/ld128/k_sinl.c \ @@ -296,18 +305,18 @@ LOCAL_SRC_FILES_arm := arm/fenv.c $(libm_arm_src_files) LOCAL_CFLAGS_arm64 := $(libm_arm64_cflags) LOCAL_C_INCLUDES_arm64 := $(libm_ld_includes) $(libm_arm64_includes) -LOCAL_SRC_FILES_arm64 := arm64/fenv.c $(libm_ld_src_files) $(libm_arm64_src_files) +LOCAL_SRC_FILES_arm64 := arm64/fenv.c $(libm_ld128_src_files) $(libm_arm64_src_files) LOCAL_C_INCLUDES_x86 := $(LOCAL_PATH)/i387 LOCAL_SRC_FILES_x86 := i387/fenv.c LOCAL_C_INCLUDES_x86_64 := $(libm_ld_includes) -LOCAL_SRC_FILES_x86_64 := amd64/fenv.c $(libm_ld_src_files) +LOCAL_SRC_FILES_x86_64 := amd64/fenv.c $(libm_ld128_src_files) LOCAL_SRC_FILES_mips := mips/fenv.c LOCAL_C_INCLUDES_mips64 := $(libm_ld_includes) -LOCAL_SRC_FILES_mips64 := mips/fenv.c $(libm_ld_src_files) +LOCAL_SRC_FILES_mips64 := mips/fenv.c $(libm_ld128_src_files) include $(BUILD_STATIC_LIBRARY) diff --git a/libm/include/complex.h b/libm/include/complex.h index 0702541c4..ff6b166f4 100644 --- a/libm/include/complex.h +++ b/libm/include/complex.h @@ -46,14 +46,39 @@ _Static_assert(__generic(_Complex_I, float _Complex, 1, 0), #define complex _Complex #define I _Complex_I +#if __ISO_C_VISIBLE >= 2011 +#ifdef __clang__ +#define CMPLX(x, y) ((double complex){ x, y }) +#define CMPLXF(x, y) ((float complex){ x, y }) +#define CMPLXL(x, y) ((long double complex){ x, y }) +#elif __GNUC_PREREQ__(4, 7) +#define CMPLX(x, y) __builtin_complex((double)(x), (double)(y)) +#define CMPLXF(x, y) __builtin_complex((float)(x), (float)(y)) +#define CMPLXL(x, y) __builtin_complex((long double)(x), (long double)(y)) +#endif +#endif /* __ISO_C_VISIBLE >= 2011 */ + __BEGIN_DECLS +#pragma GCC visibility push(default) double cabs(double complex); float cabsf(float complex); long double cabsl(long double complex); +double complex cacos(double complex); +float complex cacosf(float complex); +double complex cacosh(double complex); +float complex cacoshf(float complex); double carg(double complex); float cargf(float complex); long double cargl(long double complex); +double complex casin(double complex); +float complex casinf(float complex); +double complex casinh(double complex); +float complex casinhf(float complex); +double complex catan(double complex); +float complex catanf(float complex); +double complex catanh(double complex); +float complex catanhf(float complex); double complex ccos(double complex); float complex ccosf(float complex); double complex ccosh(double complex); @@ -87,6 +112,7 @@ float complex ctanf(float complex); double complex ctanh(double complex); float complex ctanhf(float complex); +#pragma GCC visibility pop __END_DECLS #endif /* _COMPLEX_H */ diff --git a/libm/upstream-freebsd/lib/msun/src/catrig.c b/libm/upstream-freebsd/lib/msun/src/catrig.c new file mode 100644 index 000000000..200977c0a --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/src/catrig.c @@ -0,0 +1,639 @@ +/*- + * Copyright (c) 2012 Stephen Montgomery-Smith + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include "math.h" +#include "math_private.h" + +#undef isinf +#define isinf(x) (fabs(x) == INFINITY) +#undef isnan +#define isnan(x) ((x) != (x)) +#define raise_inexact() do { volatile float junk = 1 + tiny; } while(0) +#undef signbit +#define signbit(x) (__builtin_signbit(x)) + +/* We need that DBL_EPSILON^2/128 is larger than FOUR_SQRT_MIN. */ +static const double +A_crossover = 10, /* Hull et al suggest 1.5, but 10 works better */ +B_crossover = 0.6417, /* suggested by Hull et al */ +FOUR_SQRT_MIN = 0x1p-509, /* >= 4 * sqrt(DBL_MIN) */ +QUARTER_SQRT_MAX = 0x1p509, /* <= sqrt(DBL_MAX) / 4 */ +m_e = 2.7182818284590452e0, /* 0x15bf0a8b145769.0p-51 */ +m_ln2 = 6.9314718055994531e-1, /* 0x162e42fefa39ef.0p-53 */ +pio2_hi = 1.5707963267948966e0, /* 0x1921fb54442d18.0p-52 */ +RECIP_EPSILON = 1 / DBL_EPSILON, +SQRT_3_EPSILON = 2.5809568279517849e-8, /* 0x1bb67ae8584caa.0p-78 */ +SQRT_6_EPSILON = 3.6500241499888571e-8, /* 0x13988e1409212e.0p-77 */ +SQRT_MIN = 0x1p-511; /* >= sqrt(DBL_MIN) */ + +static const volatile double +pio2_lo = 6.1232339957367659e-17; /* 0x11a62633145c07.0p-106 */ +static const volatile float +tiny = 0x1p-100; + +static double complex clog_for_large_values(double complex z); + +/* + * Testing indicates that all these functions are accurate up to 4 ULP. + * The functions casin(h) and cacos(h) are about 2.5 times slower than asinh. + * The functions catan(h) are a little under 2 times slower than atanh. + * + * The code for casinh, casin, cacos, and cacosh comes first. The code is + * rather complicated, and the four functions are highly interdependent. + * + * The code for catanh and catan comes at the end. It is much simpler than + * the other functions, and the code for these can be disconnected from the + * rest of the code. + */ + +/* + * ================================ + * | casinh, casin, cacos, cacosh | + * ================================ + */ + +/* + * The algorithm is very close to that in "Implementing the complex arcsine + * and arccosine functions using exception handling" by T. E. Hull, Thomas F. + * Fairgrieve, and Ping Tak Peter Tang, published in ACM Transactions on + * Mathematical Software, Volume 23 Issue 3, 1997, Pages 299-335, + * http://dl.acm.org/citation.cfm?id=275324. + * + * Throughout we use the convention z = x + I*y. + * + * casinh(z) = sign(x)*log(A+sqrt(A*A-1)) + I*asin(B) + * where + * A = (|z+I| + |z-I|) / 2 + * B = (|z+I| - |z-I|) / 2 = y/A + * + * These formulas become numerically unstable: + * (a) for Re(casinh(z)) when z is close to the line segment [-I, I] (that + * is, Re(casinh(z)) is close to 0); + * (b) for Im(casinh(z)) when z is close to either of the intervals + * [I, I*infinity) or (-I*infinity, -I] (that is, |Im(casinh(z))| is + * close to PI/2). + * + * These numerical problems are overcome by defining + * f(a, b) = (hypot(a, b) - b) / 2 = a*a / (hypot(a, b) + b) / 2 + * Then if A < A_crossover, we use + * log(A + sqrt(A*A-1)) = log1p((A-1) + sqrt((A-1)*(A+1))) + * A-1 = f(x, 1+y) + f(x, 1-y) + * and if B > B_crossover, we use + * asin(B) = atan2(y, sqrt(A*A - y*y)) = atan2(y, sqrt((A+y)*(A-y))) + * A-y = f(x, y+1) + f(x, y-1) + * where without loss of generality we have assumed that x and y are + * non-negative. + * + * Much of the difficulty comes because the intermediate computations may + * produce overflows or underflows. This is dealt with in the paper by Hull + * et al by using exception handling. We do this by detecting when + * computations risk underflow or overflow. The hardest part is handling the + * underflows when computing f(a, b). + * + * Note that the function f(a, b) does not appear explicitly in the paper by + * Hull et al, but the idea may be found on pages 308 and 309. Introducing the + * function f(a, b) allows us to concentrate many of the clever tricks in this + * paper into one function. + */ + +/* + * Function f(a, b, hypot_a_b) = (hypot(a, b) - b) / 2. + * Pass hypot(a, b) as the third argument. + */ +static inline double +f(double a, double b, double hypot_a_b) +{ + if (b < 0) + return ((hypot_a_b - b) / 2); + if (b == 0) + return (a / 2); + return (a * a / (hypot_a_b + b) / 2); +} + +/* + * All the hard work is contained in this function. + * x and y are assumed positive or zero, and less than RECIP_EPSILON. + * Upon return: + * rx = Re(casinh(z)) = -Im(cacos(y + I*x)). + * B_is_usable is set to 1 if the value of B is usable. + * If B_is_usable is set to 0, sqrt_A2my2 = sqrt(A*A - y*y), and new_y = y. + * If returning sqrt_A2my2 has potential to result in an underflow, it is + * rescaled, and new_y is similarly rescaled. + */ +static inline void +do_hard_work(double x, double y, double *rx, int *B_is_usable, double *B, + double *sqrt_A2my2, double *new_y) +{ + double R, S, A; /* A, B, R, and S are as in Hull et al. */ + double Am1, Amy; /* A-1, A-y. */ + + R = hypot(x, y + 1); /* |z+I| */ + S = hypot(x, y - 1); /* |z-I| */ + + /* A = (|z+I| + |z-I|) / 2 */ + A = (R + S) / 2; + /* + * Mathematically A >= 1. There is a small chance that this will not + * be so because of rounding errors. So we will make certain it is + * so. + */ + if (A < 1) + A = 1; + + if (A < A_crossover) { + /* + * Am1 = fp + fm, where fp = f(x, 1+y), and fm = f(x, 1-y). + * rx = log1p(Am1 + sqrt(Am1*(A+1))) + */ + if (y == 1 && x < DBL_EPSILON * DBL_EPSILON / 128) { + /* + * fp is of order x^2, and fm = x/2. + * A = 1 (inexactly). + */ + *rx = sqrt(x); + } else if (x >= DBL_EPSILON * fabs(y - 1)) { + /* + * Underflow will not occur because + * x >= DBL_EPSILON^2/128 >= FOUR_SQRT_MIN + */ + Am1 = f(x, 1 + y, R) + f(x, 1 - y, S); + *rx = log1p(Am1 + sqrt(Am1 * (A + 1))); + } else if (y < 1) { + /* + * fp = x*x/(1+y)/4, fm = x*x/(1-y)/4, and + * A = 1 (inexactly). + */ + *rx = x / sqrt((1 - y) * (1 + y)); + } else { /* if (y > 1) */ + /* + * A-1 = y-1 (inexactly). + */ + *rx = log1p((y - 1) + sqrt((y - 1) * (y + 1))); + } + } else { + *rx = log(A + sqrt(A * A - 1)); + } + + *new_y = y; + + if (y < FOUR_SQRT_MIN) { + /* + * Avoid a possible underflow caused by y/A. For casinh this + * would be legitimate, but will be picked up by invoking atan2 + * later on. For cacos this would not be legitimate. + */ + *B_is_usable = 0; + *sqrt_A2my2 = A * (2 / DBL_EPSILON); + *new_y = y * (2 / DBL_EPSILON); + return; + } + + /* B = (|z+I| - |z-I|) / 2 = y/A */ + *B = y / A; + *B_is_usable = 1; + + if (*B > B_crossover) { + *B_is_usable = 0; + /* + * Amy = fp + fm, where fp = f(x, y+1), and fm = f(x, y-1). + * sqrt_A2my2 = sqrt(Amy*(A+y)) + */ + if (y == 1 && x < DBL_EPSILON / 128) { + /* + * fp is of order x^2, and fm = x/2. + * A = 1 (inexactly). + */ + *sqrt_A2my2 = sqrt(x) * sqrt((A + y) / 2); + } else if (x >= DBL_EPSILON * fabs(y - 1)) { + /* + * Underflow will not occur because + * x >= DBL_EPSILON/128 >= FOUR_SQRT_MIN + * and + * x >= DBL_EPSILON^2 >= FOUR_SQRT_MIN + */ + Amy = f(x, y + 1, R) + f(x, y - 1, S); + *sqrt_A2my2 = sqrt(Amy * (A + y)); + } else if (y > 1) { + /* + * fp = x*x/(y+1)/4, fm = x*x/(y-1)/4, and + * A = y (inexactly). + * + * y < RECIP_EPSILON. So the following + * scaling should avoid any underflow problems. + */ + *sqrt_A2my2 = x * (4 / DBL_EPSILON / DBL_EPSILON) * y / + sqrt((y + 1) * (y - 1)); + *new_y = y * (4 / DBL_EPSILON / DBL_EPSILON); + } else { /* if (y < 1) */ + /* + * fm = 1-y >= DBL_EPSILON, fp is of order x^2, and + * A = 1 (inexactly). + */ + *sqrt_A2my2 = sqrt((1 - y) * (1 + y)); + } + } +} + +/* + * casinh(z) = z + O(z^3) as z -> 0 + * + * casinh(z) = sign(x)*clog(sign(x)*z) + O(1/z^2) as z -> infinity + * The above formula works for the imaginary part as well, because + * Im(casinh(z)) = sign(x)*atan2(sign(x)*y, fabs(x)) + O(y/z^3) + * as z -> infinity, uniformly in y + */ +double complex +casinh(double complex z) +{ + double x, y, ax, ay, rx, ry, B, sqrt_A2my2, new_y; + int B_is_usable; + double complex w; + + x = creal(z); + y = cimag(z); + ax = fabs(x); + ay = fabs(y); + + if (isnan(x) || isnan(y)) { + /* casinh(+-Inf + I*NaN) = +-Inf + I*NaN */ + if (isinf(x)) + return (cpack(x, y + y)); + /* casinh(NaN + I*+-Inf) = opt(+-)Inf + I*NaN */ + if (isinf(y)) + return (cpack(y, x + x)); + /* casinh(NaN + I*0) = NaN + I*0 */ + if (y == 0) + return (cpack(x + x, y)); + /* + * All other cases involving NaN return NaN + I*NaN. + * C99 leaves it optional whether to raise invalid if one of + * the arguments is not NaN, so we opt not to raise it. + */ + return (cpack(x + 0.0L + (y + 0), x + 0.0L + (y + 0))); + } + + if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) { + /* clog...() will raise inexact unless x or y is infinite. */ + if (signbit(x) == 0) + w = clog_for_large_values(z) + m_ln2; + else + w = clog_for_large_values(-z) + m_ln2; + return (cpack(copysign(creal(w), x), copysign(cimag(w), y))); + } + + /* Avoid spuriously raising inexact for z = 0. */ + if (x == 0 && y == 0) + return (z); + + /* All remaining cases are inexact. */ + raise_inexact(); + + if (ax < SQRT_6_EPSILON / 4 && ay < SQRT_6_EPSILON / 4) + return (z); + + do_hard_work(ax, ay, &rx, &B_is_usable, &B, &sqrt_A2my2, &new_y); + if (B_is_usable) + ry = asin(B); + else + ry = atan2(new_y, sqrt_A2my2); + return (cpack(copysign(rx, x), copysign(ry, y))); +} + +/* + * casin(z) = reverse(casinh(reverse(z))) + * where reverse(x + I*y) = y + I*x = I*conj(z). + */ +double complex +casin(double complex z) +{ + double complex w = casinh(cpack(cimag(z), creal(z))); + + return (cpack(cimag(w), creal(w))); +} + +/* + * cacos(z) = PI/2 - casin(z) + * but do the computation carefully so cacos(z) is accurate when z is + * close to 1. + * + * cacos(z) = PI/2 - z + O(z^3) as z -> 0 + * + * cacos(z) = -sign(y)*I*clog(z) + O(1/z^2) as z -> infinity + * The above formula works for the real part as well, because + * Re(cacos(z)) = atan2(fabs(y), x) + O(y/z^3) + * as z -> infinity, uniformly in y + */ +double complex +cacos(double complex z) +{ + double x, y, ax, ay, rx, ry, B, sqrt_A2mx2, new_x; + int sx, sy; + int B_is_usable; + double complex w; + + x = creal(z); + y = cimag(z); + sx = signbit(x); + sy = signbit(y); + ax = fabs(x); + ay = fabs(y); + + if (isnan(x) || isnan(y)) { + /* cacos(+-Inf + I*NaN) = NaN + I*opt(-)Inf */ + if (isinf(x)) + return (cpack(y + y, -INFINITY)); + /* cacos(NaN + I*+-Inf) = NaN + I*-+Inf */ + if (isinf(y)) + return (cpack(x + x, -y)); + /* cacos(0 + I*NaN) = PI/2 + I*NaN with inexact */ + if (x == 0) + return (cpack(pio2_hi + pio2_lo, y + y)); + /* + * All other cases involving NaN return NaN + I*NaN. + * C99 leaves it optional whether to raise invalid if one of + * the arguments is not NaN, so we opt not to raise it. + */ + return (cpack(x + 0.0L + (y + 0), x + 0.0L + (y + 0))); + } + + if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) { + /* clog...() will raise inexact unless x or y is infinite. */ + w = clog_for_large_values(z); + rx = fabs(cimag(w)); + ry = creal(w) + m_ln2; + if (sy == 0) + ry = -ry; + return (cpack(rx, ry)); + } + + /* Avoid spuriously raising inexact for z = 1. */ + if (x == 1 && y == 0) + return (cpack(0, -y)); + + /* All remaining cases are inexact. */ + raise_inexact(); + + if (ax < SQRT_6_EPSILON / 4 && ay < SQRT_6_EPSILON / 4) + return (cpack(pio2_hi - (x - pio2_lo), -y)); + + do_hard_work(ay, ax, &ry, &B_is_usable, &B, &sqrt_A2mx2, &new_x); + if (B_is_usable) { + if (sx == 0) + rx = acos(B); + else + rx = acos(-B); + } else { + if (sx == 0) + rx = atan2(sqrt_A2mx2, new_x); + else + rx = atan2(sqrt_A2mx2, -new_x); + } + if (sy == 0) + ry = -ry; + return (cpack(rx, ry)); +} + +/* + * cacosh(z) = I*cacos(z) or -I*cacos(z) + * where the sign is chosen so Re(cacosh(z)) >= 0. + */ +double complex +cacosh(double complex z) +{ + double complex w; + double rx, ry; + + w = cacos(z); + rx = creal(w); + ry = cimag(w); + /* cacosh(NaN + I*NaN) = NaN + I*NaN */ + if (isnan(rx) && isnan(ry)) + return (cpack(ry, rx)); + /* cacosh(NaN + I*+-Inf) = +Inf + I*NaN */ + /* cacosh(+-Inf + I*NaN) = +Inf + I*NaN */ + if (isnan(rx)) + return (cpack(fabs(ry), rx)); + /* cacosh(0 + I*NaN) = NaN + I*NaN */ + if (isnan(ry)) + return (cpack(ry, ry)); + return (cpack(fabs(ry), copysign(rx, cimag(z)))); +} + +/* + * Optimized version of clog() for |z| finite and larger than ~RECIP_EPSILON. + */ +static double complex +clog_for_large_values(double complex z) +{ + double x, y; + double ax, ay, t; + + x = creal(z); + y = cimag(z); + ax = fabs(x); + ay = fabs(y); + if (ax < ay) { + t = ax; + ax = ay; + ay = t; + } + + /* + * Avoid overflow in hypot() when x and y are both very large. + * Divide x and y by E, and then add 1 to the logarithm. This depends + * on E being larger than sqrt(2). + * Dividing by E causes an insignificant loss of accuracy; however + * this method is still poor since it is uneccessarily slow. + */ + if (ax > DBL_MAX / 2) + return (cpack(log(hypot(x / m_e, y / m_e)) + 1, atan2(y, x))); + + /* + * Avoid overflow when x or y is large. Avoid underflow when x or + * y is small. + */ + if (ax > QUARTER_SQRT_MAX || ay < SQRT_MIN) + return (cpack(log(hypot(x, y)), atan2(y, x))); + + return (cpack(log(ax * ax + ay * ay) / 2, atan2(y, x))); +} + +/* + * ================= + * | catanh, catan | + * ================= + */ + +/* + * sum_squares(x,y) = x*x + y*y (or just x*x if y*y would underflow). + * Assumes x*x and y*y will not overflow. + * Assumes x and y are finite. + * Assumes y is non-negative. + * Assumes fabs(x) >= DBL_EPSILON. + */ +static inline double +sum_squares(double x, double y) +{ + + /* Avoid underflow when y is small. */ + if (y < SQRT_MIN) + return (x * x); + + return (x * x + y * y); +} + +/* + * real_part_reciprocal(x, y) = Re(1/(x+I*y)) = x/(x*x + y*y). + * Assumes x and y are not NaN, and one of x and y is larger than + * RECIP_EPSILON. We avoid unwarranted underflow. It is important to not use + * the code creal(1/z), because the imaginary part may produce an unwanted + * underflow. + * This is only called in a context where inexact is always raised before + * the call, so no effort is made to avoid or force inexact. + */ +static inline double +real_part_reciprocal(double x, double y) +{ + double scale; + uint32_t hx, hy; + int32_t ix, iy; + + /* + * This code is inspired by the C99 document n1124.pdf, Section G.5.1, + * example 2. + */ + GET_HIGH_WORD(hx, x); + ix = hx & 0x7ff00000; + GET_HIGH_WORD(hy, y); + iy = hy & 0x7ff00000; +#define BIAS (DBL_MAX_EXP - 1) +/* XXX more guard digits are useful iff there is extra precision. */ +#define CUTOFF (DBL_MANT_DIG / 2 + 1) /* just half or 1 guard digit */ + if (ix - iy >= CUTOFF << 20 || isinf(x)) + return (1 / x); /* +-Inf -> +-0 is special */ + if (iy - ix >= CUTOFF << 20) + return (x / y / y); /* should avoid double div, but hard */ + if (ix <= (BIAS + DBL_MAX_EXP / 2 - CUTOFF) << 20) + return (x / (x * x + y * y)); + scale = 1; + SET_HIGH_WORD(scale, 0x7ff00000 - ix); /* 2**(1-ilogb(x)) */ + x *= scale; + y *= scale; + return (x / (x * x + y * y) * scale); +} + +/* + * catanh(z) = log((1+z)/(1-z)) / 2 + * = log1p(4*x / |z-1|^2) / 4 + * + I * atan2(2*y, (1-x)*(1+x)-y*y) / 2 + * + * catanh(z) = z + O(z^3) as z -> 0 + * + * catanh(z) = 1/z + sign(y)*I*PI/2 + O(1/z^3) as z -> infinity + * The above formula works for the real part as well, because + * Re(catanh(z)) = x/|z|^2 + O(x/z^4) + * as z -> infinity, uniformly in x + */ +double complex +catanh(double complex z) +{ + double x, y, ax, ay, rx, ry; + + x = creal(z); + y = cimag(z); + ax = fabs(x); + ay = fabs(y); + + /* This helps handle many cases. */ + if (y == 0 && ax <= 1) + return (cpack(atanh(x), y)); + + /* To ensure the same accuracy as atan(), and to filter out z = 0. */ + if (x == 0) + return (cpack(x, atan(y))); + + if (isnan(x) || isnan(y)) { + /* catanh(+-Inf + I*NaN) = +-0 + I*NaN */ + if (isinf(x)) + return (cpack(copysign(0, x), y + y)); + /* catanh(NaN + I*+-Inf) = sign(NaN)0 + I*+-PI/2 */ + if (isinf(y)) + return (cpack(copysign(0, x), + copysign(pio2_hi + pio2_lo, y))); + /* + * All other cases involving NaN return NaN + I*NaN. + * C99 leaves it optional whether to raise invalid if one of + * the arguments is not NaN, so we opt not to raise it. + */ + return (cpack(x + 0.0L + (y + 0), x + 0.0L + (y + 0))); + } + + if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) + return (cpack(real_part_reciprocal(x, y), + copysign(pio2_hi + pio2_lo, y))); + + if (ax < SQRT_3_EPSILON / 2 && ay < SQRT_3_EPSILON / 2) { + /* + * z = 0 was filtered out above. All other cases must raise + * inexact, but this is the only only that needs to do it + * explicitly. + */ + raise_inexact(); + return (z); + } + + if (ax == 1 && ay < DBL_EPSILON) + rx = (m_ln2 - log(ay)) / 2; + else + rx = log1p(4 * ax / sum_squares(ax - 1, ay)) / 4; + + if (ax == 1) + ry = atan2(2, -ay) / 2; + else if (ay < DBL_EPSILON) + ry = atan2(2 * ay, (1 - ax) * (1 + ax)) / 2; + else + ry = atan2(2 * ay, (1 - ax) * (1 + ax) - ay * ay) / 2; + + return (cpack(copysign(rx, x), copysign(ry, y))); +} + +/* + * catan(z) = reverse(catanh(reverse(z))) + * where reverse(x + I*y) = y + I*x = I*conj(z). + */ +double complex +catan(double complex z) +{ + double complex w = catanh(cpack(cimag(z), creal(z))); + + return (cpack(cimag(w), creal(w))); +} diff --git a/libm/upstream-freebsd/lib/msun/src/catrigf.c b/libm/upstream-freebsd/lib/msun/src/catrigf.c new file mode 100644 index 000000000..08ebef783 --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/src/catrigf.c @@ -0,0 +1,393 @@ +/*- + * Copyright (c) 2012 Stephen Montgomery-Smith + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * The algorithm is very close to that in "Implementing the complex arcsine + * and arccosine functions using exception handling" by T. E. Hull, Thomas F. + * Fairgrieve, and Ping Tak Peter Tang, published in ACM Transactions on + * Mathematical Software, Volume 23 Issue 3, 1997, Pages 299-335, + * http://dl.acm.org/citation.cfm?id=275324. + * + * See catrig.c for complete comments. + * + * XXX comments were removed automatically, and even short ones on the right + * of statements were removed (all of them), contrary to normal style. Only + * a few comments on the right of declarations remain. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include "math.h" +#include "math_private.h" + +#undef isinf +#define isinf(x) (fabsf(x) == INFINITY) +#undef isnan +#define isnan(x) ((x) != (x)) +#define raise_inexact() do { volatile float junk = 1 + tiny; } while(0) +#undef signbit +#define signbit(x) (__builtin_signbitf(x)) + +static const float +A_crossover = 10, +B_crossover = 0.6417, +FOUR_SQRT_MIN = 0x1p-61, +QUARTER_SQRT_MAX = 0x1p61, +m_e = 2.7182818285e0, /* 0xadf854.0p-22 */ +m_ln2 = 6.9314718056e-1, /* 0xb17218.0p-24 */ +pio2_hi = 1.5707962513e0, /* 0xc90fda.0p-23 */ +RECIP_EPSILON = 1 / FLT_EPSILON, +SQRT_3_EPSILON = 5.9801995673e-4, /* 0x9cc471.0p-34 */ +SQRT_6_EPSILON = 8.4572793338e-4, /* 0xddb3d7.0p-34 */ +SQRT_MIN = 0x1p-63; + +static const volatile float +pio2_lo = 7.5497899549e-8, /* 0xa22169.0p-47 */ +tiny = 0x1p-100; + +static float complex clog_for_large_values(float complex z); + +static inline float +f(float a, float b, float hypot_a_b) +{ + if (b < 0) + return ((hypot_a_b - b) / 2); + if (b == 0) + return (a / 2); + return (a * a / (hypot_a_b + b) / 2); +} + +static inline void +do_hard_work(float x, float y, float *rx, int *B_is_usable, float *B, + float *sqrt_A2my2, float *new_y) +{ + float R, S, A; + float Am1, Amy; + + R = hypotf(x, y + 1); + S = hypotf(x, y - 1); + + A = (R + S) / 2; + if (A < 1) + A = 1; + + if (A < A_crossover) { + if (y == 1 && x < FLT_EPSILON * FLT_EPSILON / 128) { + *rx = sqrtf(x); + } else if (x >= FLT_EPSILON * fabsf(y - 1)) { + Am1 = f(x, 1 + y, R) + f(x, 1 - y, S); + *rx = log1pf(Am1 + sqrtf(Am1 * (A + 1))); + } else if (y < 1) { + *rx = x / sqrtf((1 - y) * (1 + y)); + } else { + *rx = log1pf((y - 1) + sqrtf((y - 1) * (y + 1))); + } + } else { + *rx = logf(A + sqrtf(A * A - 1)); + } + + *new_y = y; + + if (y < FOUR_SQRT_MIN) { + *B_is_usable = 0; + *sqrt_A2my2 = A * (2 / FLT_EPSILON); + *new_y = y * (2 / FLT_EPSILON); + return; + } + + *B = y / A; + *B_is_usable = 1; + + if (*B > B_crossover) { + *B_is_usable = 0; + if (y == 1 && x < FLT_EPSILON / 128) { + *sqrt_A2my2 = sqrtf(x) * sqrtf((A + y) / 2); + } else if (x >= FLT_EPSILON * fabsf(y - 1)) { + Amy = f(x, y + 1, R) + f(x, y - 1, S); + *sqrt_A2my2 = sqrtf(Amy * (A + y)); + } else if (y > 1) { + *sqrt_A2my2 = x * (4 / FLT_EPSILON / FLT_EPSILON) * y / + sqrtf((y + 1) * (y - 1)); + *new_y = y * (4 / FLT_EPSILON / FLT_EPSILON); + } else { + *sqrt_A2my2 = sqrtf((1 - y) * (1 + y)); + } + } +} + +float complex +casinhf(float complex z) +{ + float x, y, ax, ay, rx, ry, B, sqrt_A2my2, new_y; + int B_is_usable; + float complex w; + + x = crealf(z); + y = cimagf(z); + ax = fabsf(x); + ay = fabsf(y); + + if (isnan(x) || isnan(y)) { + if (isinf(x)) + return (cpackf(x, y + y)); + if (isinf(y)) + return (cpackf(y, x + x)); + if (y == 0) + return (cpackf(x + x, y)); + return (cpackf(x + 0.0L + (y + 0), x + 0.0L + (y + 0))); + } + + if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) { + if (signbit(x) == 0) + w = clog_for_large_values(z) + m_ln2; + else + w = clog_for_large_values(-z) + m_ln2; + return (cpackf(copysignf(crealf(w), x), + copysignf(cimagf(w), y))); + } + + if (x == 0 && y == 0) + return (z); + + raise_inexact(); + + if (ax < SQRT_6_EPSILON / 4 && ay < SQRT_6_EPSILON / 4) + return (z); + + do_hard_work(ax, ay, &rx, &B_is_usable, &B, &sqrt_A2my2, &new_y); + if (B_is_usable) + ry = asinf(B); + else + ry = atan2f(new_y, sqrt_A2my2); + return (cpackf(copysignf(rx, x), copysignf(ry, y))); +} + +float complex +casinf(float complex z) +{ + float complex w = casinhf(cpackf(cimagf(z), crealf(z))); + + return (cpackf(cimagf(w), crealf(w))); +} + +float complex +cacosf(float complex z) +{ + float x, y, ax, ay, rx, ry, B, sqrt_A2mx2, new_x; + int sx, sy; + int B_is_usable; + float complex w; + + x = crealf(z); + y = cimagf(z); + sx = signbit(x); + sy = signbit(y); + ax = fabsf(x); + ay = fabsf(y); + + if (isnan(x) || isnan(y)) { + if (isinf(x)) + return (cpackf(y + y, -INFINITY)); + if (isinf(y)) + return (cpackf(x + x, -y)); + if (x == 0) + return (cpackf(pio2_hi + pio2_lo, y + y)); + return (cpackf(x + 0.0L + (y + 0), x + 0.0L + (y + 0))); + } + + if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) { + w = clog_for_large_values(z); + rx = fabsf(cimagf(w)); + ry = crealf(w) + m_ln2; + if (sy == 0) + ry = -ry; + return (cpackf(rx, ry)); + } + + if (x == 1 && y == 0) + return (cpackf(0, -y)); + + raise_inexact(); + + if (ax < SQRT_6_EPSILON / 4 && ay < SQRT_6_EPSILON / 4) + return (cpackf(pio2_hi - (x - pio2_lo), -y)); + + do_hard_work(ay, ax, &ry, &B_is_usable, &B, &sqrt_A2mx2, &new_x); + if (B_is_usable) { + if (sx == 0) + rx = acosf(B); + else + rx = acosf(-B); + } else { + if (sx == 0) + rx = atan2f(sqrt_A2mx2, new_x); + else + rx = atan2f(sqrt_A2mx2, -new_x); + } + if (sy == 0) + ry = -ry; + return (cpackf(rx, ry)); +} + +float complex +cacoshf(float complex z) +{ + float complex w; + float rx, ry; + + w = cacosf(z); + rx = crealf(w); + ry = cimagf(w); + if (isnan(rx) && isnan(ry)) + return (cpackf(ry, rx)); + if (isnan(rx)) + return (cpackf(fabsf(ry), rx)); + if (isnan(ry)) + return (cpackf(ry, ry)); + return (cpackf(fabsf(ry), copysignf(rx, cimagf(z)))); +} + +static float complex +clog_for_large_values(float complex z) +{ + float x, y; + float ax, ay, t; + + x = crealf(z); + y = cimagf(z); + ax = fabsf(x); + ay = fabsf(y); + if (ax < ay) { + t = ax; + ax = ay; + ay = t; + } + + if (ax > FLT_MAX / 2) + return (cpackf(logf(hypotf(x / m_e, y / m_e)) + 1, + atan2f(y, x))); + + if (ax > QUARTER_SQRT_MAX || ay < SQRT_MIN) + return (cpackf(logf(hypotf(x, y)), atan2f(y, x))); + + return (cpackf(logf(ax * ax + ay * ay) / 2, atan2f(y, x))); +} + +static inline float +sum_squares(float x, float y) +{ + + if (y < SQRT_MIN) + return (x * x); + + return (x * x + y * y); +} + +static inline float +real_part_reciprocal(float x, float y) +{ + float scale; + uint32_t hx, hy; + int32_t ix, iy; + + GET_FLOAT_WORD(hx, x); + ix = hx & 0x7f800000; + GET_FLOAT_WORD(hy, y); + iy = hy & 0x7f800000; +#define BIAS (FLT_MAX_EXP - 1) +#define CUTOFF (FLT_MANT_DIG / 2 + 1) + if (ix - iy >= CUTOFF << 23 || isinf(x)) + return (1 / x); + if (iy - ix >= CUTOFF << 23) + return (x / y / y); + if (ix <= (BIAS + FLT_MAX_EXP / 2 - CUTOFF) << 23) + return (x / (x * x + y * y)); + SET_FLOAT_WORD(scale, 0x7f800000 - ix); + x *= scale; + y *= scale; + return (x / (x * x + y * y) * scale); +} + +float complex +catanhf(float complex z) +{ + float x, y, ax, ay, rx, ry; + + x = crealf(z); + y = cimagf(z); + ax = fabsf(x); + ay = fabsf(y); + + if (y == 0 && ax <= 1) + return (cpackf(atanhf(x), y)); + + if (x == 0) + return (cpackf(x, atanf(y))); + + if (isnan(x) || isnan(y)) { + if (isinf(x)) + return (cpackf(copysignf(0, x), y + y)); + if (isinf(y)) + return (cpackf(copysignf(0, x), + copysignf(pio2_hi + pio2_lo, y))); + return (cpackf(x + 0.0L + (y + 0), x + 0.0L + (y + 0))); + } + + if (ax > RECIP_EPSILON || ay > RECIP_EPSILON) + return (cpackf(real_part_reciprocal(x, y), + copysignf(pio2_hi + pio2_lo, y))); + + if (ax < SQRT_3_EPSILON / 2 && ay < SQRT_3_EPSILON / 2) { + raise_inexact(); + return (z); + } + + if (ax == 1 && ay < FLT_EPSILON) + rx = (m_ln2 - logf(ay)) / 2; + else + rx = log1pf(4 * ax / sum_squares(ax - 1, ay)) / 4; + + if (ax == 1) + ry = atan2f(2, -ay) / 2; + else if (ay < FLT_EPSILON) + ry = atan2f(2 * ay, (1 - ax) * (1 + ax)) / 2; + else + ry = atan2f(2 * ay, (1 - ax) * (1 + ax) - ay * ay) / 2; + + return (cpackf(copysignf(rx, x), copysignf(ry, y))); +} + +float complex +catanf(float complex z) +{ + float complex w = catanhf(cpackf(cimagf(z), crealf(z))); + + return (cpackf(cimagf(w), crealf(w))); +} diff --git a/tests/Android.mk b/tests/Android.mk index 6eeeacbf4..2db088d3f 100644 --- a/tests/Android.mk +++ b/tests/Android.mk @@ -54,6 +54,7 @@ test_cppflags = \ libBionicStandardTests_src_files := \ arpa_inet_test.cpp \ buffer_tests.cpp \ + complex_test.cpp \ ctype_test.cpp \ dirent_test.cpp \ eventfd_test.cpp \ diff --git a/tests/complex_test.cpp b/tests/complex_test.cpp new file mode 100644 index 000000000..47964f6b9 --- /dev/null +++ b/tests/complex_test.cpp @@ -0,0 +1,260 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +// libc++ actively gets in the way of including from C++, so we +// have to declare the complex math functions ourselves. +// (libc++ also seems to have really bad implementations of its own that ignore +// the intricacies of floating point math.) +// http://llvm.org/bugs/show_bug.cgi?id=21504 + +#include // For M_PI. + +extern "C" double cabs(double _Complex); +TEST(complex, cabs) { + ASSERT_EQ(0.0, cabs(0)); +} + +extern "C" float cabsf(float _Complex); +TEST(complex, cabsf) { + ASSERT_EQ(0.0, cabsf(0)); +} + +extern "C" long double cabsl(long double _Complex); +TEST(complex, cabsl) { + ASSERT_EQ(0.0, cabsl(0)); +} + +extern "C" double _Complex cacos(double _Complex); +TEST(complex, cacos) { + ASSERT_EQ(M_PI/2.0, cacos(0.0)); +} + +extern "C" float _Complex cacosf(float _Complex); +TEST(complex, cacosf) { + ASSERT_EQ(static_cast(M_PI)/2.0f, cacosf(0.0)); +} + +extern "C" double _Complex cacosh(double _Complex); +TEST(complex, cacosh) { + ASSERT_EQ(0.0, cacosh(1.0)); +} + +extern "C" float _Complex cacoshf(float _Complex); +TEST(complex, cacoshf) { + ASSERT_EQ(0.0, cacoshf(1.0)); +} + +extern "C" double carg(double _Complex); +TEST(complex, carg) { + ASSERT_EQ(0.0, carg(0)); +} + +extern "C" float cargf(float _Complex); +TEST(complex, cargf) { + ASSERT_EQ(0.0, cargf(0)); +} + +extern "C" long double cargl(long double _Complex); +TEST(complex, cargl) { + ASSERT_EQ(0.0, cargl(0)); +} + +extern "C" double _Complex casin(double _Complex); +TEST(complex, casin) { + ASSERT_EQ(0.0, casin(0)); +} + +extern "C" float _Complex casinf(float _Complex); +TEST(complex, casinf) { + ASSERT_EQ(0.0, casinf(0)); +} + +extern "C" double _Complex casinh(double _Complex); +TEST(complex, casinh) { + ASSERT_EQ(0.0, casinh(0)); +} + +extern "C" float _Complex casinhf(float _Complex); +TEST(complex, casinhf) { + ASSERT_EQ(0.0, casinhf(0)); +} + +extern "C" double _Complex catan(double _Complex); +TEST(complex, catan) { + ASSERT_EQ(0.0, catan(0)); +} + +extern "C" float _Complex catanf(float _Complex); +TEST(complex, catanf) { + ASSERT_EQ(0.0, catanf(0)); +} + +extern "C" double _Complex catanh(double _Complex); +TEST(complex, catanh) { + ASSERT_EQ(0.0, catanh(0)); +} + +extern "C" float _Complex catanhf(float _Complex); +TEST(complex, catanhf) { + ASSERT_EQ(0.0, catanhf(0)); +} + +extern "C" double _Complex ccos(double _Complex); +TEST(complex, ccos) { + ASSERT_EQ(1.0, ccos(0)); +} + +extern "C" float _Complex ccosf(float _Complex); +TEST(complex, ccosf) { + ASSERT_EQ(1.0, ccosf(0)); +} + +extern "C" double _Complex ccosh(double _Complex); +TEST(complex, ccosh) { + ASSERT_EQ(1.0, ccosh(0)); +} + +extern "C" float _Complex ccoshf(float _Complex); +TEST(complex, ccoshf) { + ASSERT_EQ(1.0, ccoshf(0)); +} + +extern "C" double _Complex cexp(double _Complex); +TEST(complex, cexp) { + ASSERT_EQ(1.0, cexp(0)); +} + +extern "C" float _Complex cexpf(float _Complex); +TEST(complex, cexpf) { + ASSERT_EQ(1.0, cexpf(0)); +} + +extern "C" double cimag(double _Complex); +TEST(complex, cimag) { + ASSERT_EQ(0.0, cimag(0)); +} + +extern "C" float cimagf(float _Complex); +TEST(complex, cimagf) { + ASSERT_EQ(0.0f, cimagf(0)); +} + +extern "C" long double cimagl(long double _Complex); +TEST(complex, cimagl) { + ASSERT_EQ(0.0, cimagl(0)); +} + +extern "C" double _Complex conj(double _Complex); +TEST(complex, conj) { + ASSERT_EQ(0.0, conj(0)); +} + +extern "C" float _Complex conjf(float _Complex); +TEST(complex, conjf) { + ASSERT_EQ(0.0f, conjf(0)); +} + +extern "C" long double _Complex conjl(long double _Complex); +TEST(complex, conjl) { + ASSERT_EQ(0.0, conjl(0)); +} + +extern "C" double _Complex cproj(double _Complex); +TEST(complex, cproj) { + ASSERT_EQ(0.0, cproj(0)); +} + +extern "C" float _Complex cprojf(float _Complex); +TEST(complex, cprojf) { + ASSERT_EQ(0.0f, cprojf(0)); +} + +extern "C" long double _Complex cprojl(long double _Complex); +TEST(complex, cprojl) { + ASSERT_EQ(0.0, cprojl(0)); +} + +extern "C" double creal(double _Complex); +TEST(complex, creal) { + ASSERT_EQ(0.0, creal(0)); +} + +extern "C" float crealf(float _Complex); +TEST(complex, crealf) { + ASSERT_EQ(0.0f, crealf(0)); +} + +extern "C" long double creall(long double _Complex); +TEST(complex, creall) { + ASSERT_EQ(0.0, creall(0)); +} + +extern "C" double _Complex csin(double _Complex); +TEST(complex, csin) { + ASSERT_EQ(0.0, csin(0)); +} + +extern "C" float _Complex csinf(float _Complex); +TEST(complex, csinf) { + ASSERT_EQ(0.0, csinf(0)); +} + +extern "C" double _Complex csinh(double _Complex); +TEST(complex, csinh) { + ASSERT_EQ(0.0, csinh(0)); +} + +extern "C" float _Complex csinhf(float _Complex); +TEST(complex, csinhf) { + ASSERT_EQ(0.0, csinhf(0)); +} + +extern "C" double _Complex csqrt(double _Complex); +TEST(complex, csqrt) { + ASSERT_EQ(0.0, csqrt(0)); +} + +extern "C" float _Complex csqrtf(float _Complex); +TEST(complex, csqrtf) { + ASSERT_EQ(0.0f, csqrt(0)); +} + +extern "C" long double _Complex csqrtl(long double _Complex); +TEST(complex, csqrtl) { + ASSERT_EQ(0.0, csqrtl(0)); +} + +extern "C" double _Complex ctan(double _Complex); +TEST(complex, ctan) { + ASSERT_EQ(0.0, ctan(0)); +} + +extern "C" float _Complex ctanf(float _Complex); +TEST(complex, ctanf) { + ASSERT_EQ(0.0, ctanf(0)); +} + +extern "C" double _Complex ctanh(double _Complex); +TEST(complex, ctanh) { + ASSERT_EQ(0.0, ctanh(0)); +} + +extern "C" float _Complex ctanhf(float _Complex); +TEST(complex, ctanhf) { + ASSERT_EQ(0.0, ctanhf(0)); +} From 65dff075a6d6ff46c05bacbce4e3f0794c747b59 Mon Sep 17 00:00:00 2001 From: Shu Zhang Date: Wed, 23 Jul 2014 16:59:22 +0800 Subject: [PATCH 73/89] libm: arm: Add arm specific floor() optimization Add arm specific floor() implementation which avoids VMSR and VMRS instructions. Change-Id: Ibd4cd7147aa2f98c9b5bbaf74948843ea619dba4 --- libm/Android.mk | 33 ++++++++++- libm/arm/s_floor.S | 142 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 libm/arm/s_floor.S diff --git a/libm/Android.mk b/libm/Android.mk index 27967c5e4..dc983e78e 100644 --- a/libm/Android.mk +++ b/libm/Android.mk @@ -124,7 +124,6 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_fdim.c \ upstream-freebsd/lib/msun/src/s_finite.c \ upstream-freebsd/lib/msun/src/s_finitef.c \ - upstream-freebsd/lib/msun/src/s_floor.c \ upstream-freebsd/lib/msun/src/s_floorf.c \ upstream-freebsd/lib/msun/src/s_fma.c \ upstream-freebsd/lib/msun/src/s_fmaf.c \ @@ -238,6 +237,35 @@ libm_ld128_src_files += \ upstream-freebsd/lib/msun/ld128/s_logl.c \ upstream-freebsd/lib/msun/ld128/s_nanl.c \ +# s_floor.S requires neon instructions. +ifdef TARGET_2ND_ARCH +arch_variant := $(TARGET_2ND_ARCH_VARIANT) +else +arch_variant := $(TARGET_ARCH_VARIANT) +endif + +# Use the C version on armv7-a since it doesn't support neon instructions. +ifeq ($(arch_variant),armv7-a) +LOCAL_SRC_FILES_arm += upstream-freebsd/lib/msun/src/s_floor.c +else +LOCAL_SRC_FILES_arm += arm/s_floor.S +endif + +LOCAL_SRC_FILES_arm64 += \ + upstream-freebsd/lib/msun/src/s_floor.c \ + +LOCAL_SRC_FILES_mips += \ + upstream-freebsd/lib/msun/src/s_floor.c \ + +LOCAL_SRC_FILES_mips64 += \ + upstream-freebsd/lib/msun/src/s_floor.c \ + +LOCAL_SRC_FILES_x86 += \ + upstream-freebsd/lib/msun/src/s_floor.c \ + +LOCAL_SRC_FILES_x86_64 += \ + upstream-freebsd/lib/msun/src/s_floor.c \ + # TODO: re-enable i387/e_sqrtf.S for x86, and maybe others. libm_common_cflags := \ @@ -251,6 +279,9 @@ libm_common_cflags := \ -Wno-unknown-pragmas \ -fvisibility=hidden \ +LOCAL_ASFLAGS := \ + -Ibionic/libc \ + # Workaround the GCC "(long)fn -> lfn" optimization bug which will result in # self recursions for lrint, lrintf, and lrintl. # BUG: 14225968 diff --git a/libm/arm/s_floor.S b/libm/arm/s_floor.S new file mode 100644 index 000000000..44053581e --- /dev/null +++ b/libm/arm/s_floor.S @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2013-2014, NVIDIA Corporation. All rights reserved. + * Johnny Qiu + * Shu Zhang + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of The Linux Foundation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +ENTRY(floor) /* x in r0, r1 */ + + and r3, r1, #0x80000000 /* sign(x) */ + bic r1, r1, #0x80000000 /* x = abs(x) */ + + /* extract exp of x */ + lsr r2, r1, #20 + sub r2, r2, #0x3fc + subs r2, r2, #0x3 /* r2 <- exp */ + + /* |x| < 1.0? */ + blt .Lx_lt_one + + /* x < 0? */ + cmp r3, #0 + bne .Lclr_frac_neg + + /* |x| <= 2^20? */ + cmp r2, #20 + ble .Lclr_frac_r1 + + /* |x| < 2^52? */ + cmp r2, #52 + blt .Lclr_frac_r0 + + /* return x */ + bx lr + +.Lclr_frac_r1: + rsb r2, r2, #20 + lsr r1, r1, r2 + lsl r1, r1, r2 + mov r0, #0 + bx lr + +.Lclr_frac_r0: + rsb r2, r2, #52 + lsr r0, r0, r2 + lsl r0, r0, r2 + bx lr + +.Lclr_frac_neg: + /* |x| <= 2^20? */ + cmp r2, #20 + ble .Lclr_frac_r1_neg + + /* |x| < 2^52? */ + cmp r2, #52 + blt .Lclr_frac_r0_neg + + /* return x */ + orr r1, r1, #0x80000000 + bx lr + +.Lclr_frac_r1_neg: + rsb r2, r2, #20 + mov r3, #1 + lsl r3, r3, r2 + sub r3, r3, #1 + and r3, r1, r3 + orr r3, r3, r0 + lsr r1, r1, r2 + lsl r1, r1, r2 + mov r0, #0 + b .Lreturn_x_neg + +.Lclr_frac_r0_neg: + rsb r2, r2, #52 + mov r3, #1 + lsl r3, r3, r2 + sub r3, r3, #1 + and r3, r0, r3 + lsr r0, r0, r2 + lsl r0, r0, r2 + b .Lreturn_x_neg + +.Lx_lt_one: + /* x == +-0? */ + cmp r0, #0 + cmpeq r1, #0 + orreq r1, r1, r3 + bxeq lr + + /* (x > 0) ? 0 : -1 */ + mov r1, #0x00100000 + mov r0, #0 + cmp r3, #0 + movne r1, #0xc0000000 + sub r1, r1, #0x00100000 + bx lr + +.Lreturn_x_neg: + cmp r3, #0 + orr r1, r1, #0x80000000 + bxeq lr + + vmov d16, r0, r1 + vmov.f64 d18, #1.0 + vsub.f64 d16, d16, d18 + vmov r0, r1, d16 + bx lr + +END(floor) + +#if LDBL_MANT_DIG == 53 + .weak floorl + .equ floorl,floor +#endif From 2b8a776d6e8e7d0854d9a70898ce9c827338f2ca Mon Sep 17 00:00:00 2001 From: Amaury Le Leyzour Date: Tue, 17 Feb 2015 15:05:51 -0800 Subject: [PATCH 74/89] Add aarch64 instructions wherever possible fma, sqrt and various rounding functions have aarch64 instruction equivalent Change-Id: I1284f31b9f78f914281e5563b8d44db8362b627d --- libm/Android.mk | 107 ++++++++++++++++++++++++++++++++++++++------- libm/arm64/ceil.S | 27 ++++++++++++ libm/arm64/floor.S | 27 ++++++++++++ libm/arm64/fma.S | 27 ++++++++++++ libm/arm64/lrint.S | 36 +++++++++++++++ libm/arm64/rint.S | 27 ++++++++++++ libm/arm64/sqrt.S | 27 ++++++++++++ libm/arm64/trunc.S | 27 ++++++++++++ 8 files changed, 288 insertions(+), 17 deletions(-) create mode 100644 libm/arm64/ceil.S create mode 100644 libm/arm64/floor.S create mode 100644 libm/arm64/fma.S create mode 100644 libm/arm64/lrint.S create mode 100644 libm/arm64/rint.S create mode 100644 libm/arm64/sqrt.S create mode 100644 libm/arm64/trunc.S diff --git a/libm/Android.mk b/libm/Android.mk index dc983e78e..b811b8a60 100644 --- a/libm/Android.mk +++ b/libm/Android.mk @@ -87,8 +87,6 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_cbrtf.c \ upstream-freebsd/lib/msun/src/s_ccosh.c \ upstream-freebsd/lib/msun/src/s_ccoshf.c \ - upstream-freebsd/lib/msun/src/s_ceil.c \ - upstream-freebsd/lib/msun/src/s_ceilf.c \ upstream-freebsd/lib/msun/src/s_cexp.c \ upstream-freebsd/lib/msun/src/s_cexpf.c \ upstream-freebsd/lib/msun/src/s_cimag.c \ @@ -124,9 +122,6 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_fdim.c \ upstream-freebsd/lib/msun/src/s_finite.c \ upstream-freebsd/lib/msun/src/s_finitef.c \ - upstream-freebsd/lib/msun/src/s_floorf.c \ - upstream-freebsd/lib/msun/src/s_fma.c \ - upstream-freebsd/lib/msun/src/s_fmaf.c \ upstream-freebsd/lib/msun/src/s_fmax.c \ upstream-freebsd/lib/msun/src/s_fmaxf.c \ upstream-freebsd/lib/msun/src/s_fmin.c \ @@ -135,16 +130,12 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_frexpf.c \ upstream-freebsd/lib/msun/src/s_ilogb.c \ upstream-freebsd/lib/msun/src/s_ilogbf.c \ - upstream-freebsd/lib/msun/src/s_llrint.c \ - upstream-freebsd/lib/msun/src/s_llrintf.c \ upstream-freebsd/lib/msun/src/s_llround.c \ upstream-freebsd/lib/msun/src/s_llroundf.c \ upstream-freebsd/lib/msun/src/s_log1p.c \ upstream-freebsd/lib/msun/src/s_log1pf.c \ upstream-freebsd/lib/msun/src/s_logb.c \ upstream-freebsd/lib/msun/src/s_logbf.c \ - upstream-freebsd/lib/msun/src/s_lrint.c \ - upstream-freebsd/lib/msun/src/s_lrintf.c \ upstream-freebsd/lib/msun/src/s_lround.c \ upstream-freebsd/lib/msun/src/s_lroundf.c \ upstream-freebsd/lib/msun/src/s_modf.c \ @@ -155,8 +146,6 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_nextafterf.c \ upstream-freebsd/lib/msun/src/s_remquo.c \ upstream-freebsd/lib/msun/src/s_remquof.c \ - upstream-freebsd/lib/msun/src/s_rint.c \ - upstream-freebsd/lib/msun/src/s_rintf.c \ upstream-freebsd/lib/msun/src/s_round.c \ upstream-freebsd/lib/msun/src/s_roundf.c \ upstream-freebsd/lib/msun/src/s_scalbln.c \ @@ -171,8 +160,6 @@ libm_common_src_files += \ upstream-freebsd/lib/msun/src/s_tanh.c \ upstream-freebsd/lib/msun/src/s_tanhf.c \ upstream-freebsd/lib/msun/src/s_tgammaf.c \ - upstream-freebsd/lib/msun/src/s_trunc.c \ - upstream-freebsd/lib/msun/src/s_truncf.c \ upstream-freebsd/lib/msun/src/w_cabs.c \ upstream-freebsd/lib/msun/src/w_cabsf.c \ upstream-freebsd/lib/msun/src/w_cabsl.c \ @@ -237,6 +224,28 @@ libm_ld128_src_files += \ upstream-freebsd/lib/msun/ld128/s_logl.c \ upstream-freebsd/lib/msun/ld128/s_nanl.c \ +# Arch specific optimizations. + +# ----------------------------------------------------------------------------- +# arm +# ----------------------------------------------------------------------------- +LOCAL_SRC_FILES_arm += \ + upstream-freebsd/lib/msun/src/e_sqrt.c \ + upstream-freebsd/lib/msun/src/e_sqrtf.c \ + upstream-freebsd/lib/msun/src/s_ceil.c \ + upstream-freebsd/lib/msun/src/s_ceilf.c \ + upstream-freebsd/lib/msun/src/s_fma.c \ + upstream-freebsd/lib/msun/src/s_fmaf.c \ + upstream-freebsd/lib/msun/src/s_floorf.c \ + upstream-freebsd/lib/msun/src/s_llrint.c \ + upstream-freebsd/lib/msun/src/s_llrintf.c \ + upstream-freebsd/lib/msun/src/s_lrint.c \ + upstream-freebsd/lib/msun/src/s_lrintf.c \ + upstream-freebsd/lib/msun/src/s_rint.c \ + upstream-freebsd/lib/msun/src/s_rintf.c \ + upstream-freebsd/lib/msun/src/s_trunc.c \ + upstream-freebsd/lib/msun/src/s_truncf.c \ + # s_floor.S requires neon instructions. ifdef TARGET_2ND_ARCH arch_variant := $(TARGET_2ND_ARCH_VARIANT) @@ -251,20 +260,84 @@ else LOCAL_SRC_FILES_arm += arm/s_floor.S endif +# ----------------------------------------------------------------------------- +# arm64 +# ----------------------------------------------------------------------------- LOCAL_SRC_FILES_arm64 += \ - upstream-freebsd/lib/msun/src/s_floor.c \ + arm64/ceil.S \ + arm64/fma.S \ + arm64/floor.S \ + arm64/lrint.S \ + arm64/rint.S \ + arm64/sqrt.S \ + arm64/trunc.S \ -LOCAL_SRC_FILES_mips += \ +# ----------------------------------------------------------------------------- +# mips +# ----------------------------------------------------------------------------- +libm_mips_arch_files := \ + mips/fenv.c \ + upstream-freebsd/lib/msun/src/e_sqrt.c \ + upstream-freebsd/lib/msun/src/e_sqrtf.c \ + upstream-freebsd/lib/msun/src/s_ceil.c \ + upstream-freebsd/lib/msun/src/s_ceilf.c \ + upstream-freebsd/lib/msun/src/s_fma.c \ + upstream-freebsd/lib/msun/src/s_fmaf.c \ upstream-freebsd/lib/msun/src/s_floor.c \ + upstream-freebsd/lib/msun/src/s_floorf.c \ + upstream-freebsd/lib/msun/src/s_llrint.c \ + upstream-freebsd/lib/msun/src/s_llrintf.c \ + upstream-freebsd/lib/msun/src/s_lrint.c \ + upstream-freebsd/lib/msun/src/s_lrintf.c \ + upstream-freebsd/lib/msun/src/s_rint.c \ + upstream-freebsd/lib/msun/src/s_rintf.c \ + upstream-freebsd/lib/msun/src/s_trunc.c \ + upstream-freebsd/lib/msun/src/s_truncf.c \ -LOCAL_SRC_FILES_mips64 += \ - upstream-freebsd/lib/msun/src/s_floor.c \ +LOCAL_SRC_FILES_mips += $(libm_mips_arch_files) +LOCAL_SRC_FILES_mips64 += $(libm_mips_arch_files) +# ----------------------------------------------------------------------------- +# x86 +# ----------------------------------------------------------------------------- LOCAL_SRC_FILES_x86 += \ + upstream-freebsd/lib/msun/src/e_sqrt.c \ + upstream-freebsd/lib/msun/src/e_sqrtf.c \ + upstream-freebsd/lib/msun/src/s_ceil.c \ + upstream-freebsd/lib/msun/src/s_ceilf.c \ + upstream-freebsd/lib/msun/src/s_fma.c \ + upstream-freebsd/lib/msun/src/s_fmaf.c \ upstream-freebsd/lib/msun/src/s_floor.c \ + upstream-freebsd/lib/msun/src/s_floorf.c \ + upstream-freebsd/lib/msun/src/s_llrint.c \ + upstream-freebsd/lib/msun/src/s_llrintf.c \ + upstream-freebsd/lib/msun/src/s_lrint.c \ + upstream-freebsd/lib/msun/src/s_lrintf.c \ + upstream-freebsd/lib/msun/src/s_rint.c \ + upstream-freebsd/lib/msun/src/s_rintf.c \ + upstream-freebsd/lib/msun/src/s_trunc.c \ + upstream-freebsd/lib/msun/src/s_truncf.c \ +# ----------------------------------------------------------------------------- +# x86_64 +# ----------------------------------------------------------------------------- LOCAL_SRC_FILES_x86_64 += \ + upstream-freebsd/lib/msun/src/e_sqrt.c \ + upstream-freebsd/lib/msun/src/e_sqrtf.c \ + upstream-freebsd/lib/msun/src/s_ceil.c \ + upstream-freebsd/lib/msun/src/s_ceilf.c \ + upstream-freebsd/lib/msun/src/s_fma.c \ + upstream-freebsd/lib/msun/src/s_fmaf.c \ upstream-freebsd/lib/msun/src/s_floor.c \ + upstream-freebsd/lib/msun/src/s_floorf.c \ + upstream-freebsd/lib/msun/src/s_llrint.c \ + upstream-freebsd/lib/msun/src/s_llrintf.c \ + upstream-freebsd/lib/msun/src/s_lrint.c \ + upstream-freebsd/lib/msun/src/s_lrintf.c \ + upstream-freebsd/lib/msun/src/s_rint.c \ + upstream-freebsd/lib/msun/src/s_rintf.c \ + upstream-freebsd/lib/msun/src/s_trunc.c \ + upstream-freebsd/lib/msun/src/s_truncf.c \ # TODO: re-enable i387/e_sqrtf.S for x86, and maybe others. diff --git a/libm/arm64/ceil.S b/libm/arm64/ceil.S new file mode 100644 index 000000000..006c9887a --- /dev/null +++ b/libm/arm64/ceil.S @@ -0,0 +1,27 @@ +/* + * Copyright 2015, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +ENTRY(ceil) + frintP d0, d0 + ret +END(ceil) + +ENTRY(ceilf) + frintP s0, s0 + ret +END(ceilf) diff --git a/libm/arm64/floor.S b/libm/arm64/floor.S new file mode 100644 index 000000000..2d792e555 --- /dev/null +++ b/libm/arm64/floor.S @@ -0,0 +1,27 @@ +/* + * Copyright 2015, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +ENTRY(floor) + frintM d0, d0 + ret +END(floor) + +ENTRY(floorf) + frintM s0, s0 + ret +END(floorf) diff --git a/libm/arm64/fma.S b/libm/arm64/fma.S new file mode 100644 index 000000000..e64e8077e --- /dev/null +++ b/libm/arm64/fma.S @@ -0,0 +1,27 @@ +/* + * Copyright 2015, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +ENTRY(fma) + fmadd d0, d0, d1, d2 + ret +END(fma) + +ENTRY(fmaf) + fmadd s0, s0, s1, s2 + ret +END(fmaf) diff --git a/libm/arm64/lrint.S b/libm/arm64/lrint.S new file mode 100644 index 000000000..69cc10c3e --- /dev/null +++ b/libm/arm64/lrint.S @@ -0,0 +1,36 @@ +/* + * Copyright 2015, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +ENTRY(lrint) + frintX d0, d0 + fcvtzs x0, d0 + ret +END(lrint) + +ENTRY(lrintf) + frintX s0, s0 + fcvtzs x0, s0 + ret +END(lrintf) + +// sizeof(long) and sizeof(long long) are the same for aarch64 +.weak llrint +.equ llrint,lrint + +.weak llrintf +.equ llrintf,lrintf diff --git a/libm/arm64/rint.S b/libm/arm64/rint.S new file mode 100644 index 000000000..0820c2211 --- /dev/null +++ b/libm/arm64/rint.S @@ -0,0 +1,27 @@ +/* + * Copyright 2015, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +ENTRY(rint) + frintX d0, d0 + ret +END(rint) + +ENTRY(rintf) + frintX s0, s0 + ret +END(rintf) diff --git a/libm/arm64/sqrt.S b/libm/arm64/sqrt.S new file mode 100644 index 000000000..fe0020bd6 --- /dev/null +++ b/libm/arm64/sqrt.S @@ -0,0 +1,27 @@ +/* + * Copyright 2015, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +ENTRY(sqrt) + fsqrt d0, d0 + ret +END(sqrt) + +ENTRY(sqrtf) + fsqrt s0, s0 + ret +END(sqrtf) diff --git a/libm/arm64/trunc.S b/libm/arm64/trunc.S new file mode 100644 index 000000000..329c08d12 --- /dev/null +++ b/libm/arm64/trunc.S @@ -0,0 +1,27 @@ +/* + * Copyright 2015, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +ENTRY(trunc) + frintZ d0, d0 + ret +END(trunc) + +ENTRY(truncf) + frintZ s0, s0 + ret +END(truncf) From 3e8f62d924eccd9d1df7bd6187e70437ca8b1a8d Mon Sep 17 00:00:00 2001 From: Steve Kondik Date: Sun, 12 Jul 2015 14:35:27 -0700 Subject: [PATCH 75/89] libm: Fix makefiles * Generalize some of the math routines for all v7/v8s. * Disable the pow optimization as it fails CTS. Change-Id: I497651a72d3c298f7648831611d16e4f99245a0c --- libc/arch-arm64/arm64.mk | 2 +- libm/Android.mk | 98 +++++++++---------- libm/upstream-freebsd/lib/msun/src/e_pow.c | 35 +------ libm/upstream-freebsd/lib/msun/src/k_cos.c | 2 +- libm/upstream-freebsd/lib/msun/src/k_sin.c | 2 +- .../lib/msun/src/math_private.h | 4 +- 6 files changed, 52 insertions(+), 91 deletions(-) diff --git a/libc/arch-arm64/arm64.mk b/libc/arch-arm64/arm64.mk index bb6ca6303..a4c32efe3 100644 --- a/libc/arch-arm64/arm64.mk +++ b/libc/arch-arm64/arm64.mk @@ -61,6 +61,6 @@ ifeq ($(wildcard $(cpu_variant_mk)),) $(error "TARGET_CPU_VARIANT not set or set to an unknown value. Possible values are generic, denver64. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.") endif include $(cpu_variant_mk) -libc_common_additional_dependencies += $(cpu_variank_mk) +libc_common_additional_dependencies += $(cpu_variant_mk) cpu_variant_mk := diff --git a/libm/Android.mk b/libm/Android.mk index b811b8a60..40098b720 100644 --- a/libm/Android.mk +++ b/libm/Android.mk @@ -229,9 +229,7 @@ libm_ld128_src_files += \ # ----------------------------------------------------------------------------- # arm # ----------------------------------------------------------------------------- -LOCAL_SRC_FILES_arm += \ - upstream-freebsd/lib/msun/src/e_sqrt.c \ - upstream-freebsd/lib/msun/src/e_sqrtf.c \ +libm_arm_arch_files += \ upstream-freebsd/lib/msun/src/s_ceil.c \ upstream-freebsd/lib/msun/src/s_ceilf.c \ upstream-freebsd/lib/msun/src/s_fma.c \ @@ -255,15 +253,30 @@ endif # Use the C version on armv7-a since it doesn't support neon instructions. ifeq ($(arch_variant),armv7-a) -LOCAL_SRC_FILES_arm += upstream-freebsd/lib/msun/src/s_floor.c +libm_arm_arch_files += \ + upstream-freebsd/lib/msun/src/s_cos.c \ + upstream-freebsd/lib/msun/src/s_sin.c \ + upstream-freebsd/lib/msun/src/e_sqrt.c \ + upstream-freebsd/lib/msun/src/e_sqrtf.c \ + upstream-freebsd/lib/msun/src/s_floor.c \ + else -LOCAL_SRC_FILES_arm += arm/s_floor.S +libm_arm_arch_files += \ + arm/s_cos.S \ + arm/s_sin.S \ + arm/e_sqrt.S \ + arm/e_sqrtf.S \ + arm/s_floor.S \ + +libm_arm_arch_cflags += -DQCOM_NEON_OPTIMIZATION -fno-if-conversion endif # ----------------------------------------------------------------------------- # arm64 # ----------------------------------------------------------------------------- -LOCAL_SRC_FILES_arm64 += \ +libm_arm64_arch_files += \ + upstream-freebsd/lib/msun/src/s_cos.c \ + upstream-freebsd/lib/msun/src/s_sin.c \ arm64/ceil.S \ arm64/fma.S \ arm64/floor.S \ @@ -272,15 +285,18 @@ LOCAL_SRC_FILES_arm64 += \ arm64/sqrt.S \ arm64/trunc.S \ +libm_arm64_arch_cflags += -DQCOM_NEON_OPTIMIZATION + # ----------------------------------------------------------------------------- # mips # ----------------------------------------------------------------------------- -libm_mips_arch_files := \ +libm_mips_arch_files += \ mips/fenv.c \ upstream-freebsd/lib/msun/src/e_sqrt.c \ upstream-freebsd/lib/msun/src/e_sqrtf.c \ upstream-freebsd/lib/msun/src/s_ceil.c \ upstream-freebsd/lib/msun/src/s_ceilf.c \ + upstream-freebsd/lib/msun/src/s_cos.c \ upstream-freebsd/lib/msun/src/s_fma.c \ upstream-freebsd/lib/msun/src/s_fmaf.c \ upstream-freebsd/lib/msun/src/s_floor.c \ @@ -291,20 +307,20 @@ libm_mips_arch_files := \ upstream-freebsd/lib/msun/src/s_lrintf.c \ upstream-freebsd/lib/msun/src/s_rint.c \ upstream-freebsd/lib/msun/src/s_rintf.c \ + upstream-freebsd/lib/msun/src/s_sin.c \ upstream-freebsd/lib/msun/src/s_trunc.c \ upstream-freebsd/lib/msun/src/s_truncf.c \ -LOCAL_SRC_FILES_mips += $(libm_mips_arch_files) -LOCAL_SRC_FILES_mips64 += $(libm_mips_arch_files) # ----------------------------------------------------------------------------- # x86 # ----------------------------------------------------------------------------- -LOCAL_SRC_FILES_x86 += \ +libm_x86_arch_files += \ upstream-freebsd/lib/msun/src/e_sqrt.c \ upstream-freebsd/lib/msun/src/e_sqrtf.c \ upstream-freebsd/lib/msun/src/s_ceil.c \ upstream-freebsd/lib/msun/src/s_ceilf.c \ + upstream-freebsd/lib/msun/src/s_cos.c \ upstream-freebsd/lib/msun/src/s_fma.c \ upstream-freebsd/lib/msun/src/s_fmaf.c \ upstream-freebsd/lib/msun/src/s_floor.c \ @@ -315,17 +331,19 @@ LOCAL_SRC_FILES_x86 += \ upstream-freebsd/lib/msun/src/s_lrintf.c \ upstream-freebsd/lib/msun/src/s_rint.c \ upstream-freebsd/lib/msun/src/s_rintf.c \ + upstream-freebsd/lib/msun/src/s_sin.c \ upstream-freebsd/lib/msun/src/s_trunc.c \ upstream-freebsd/lib/msun/src/s_truncf.c \ # ----------------------------------------------------------------------------- # x86_64 # ----------------------------------------------------------------------------- -LOCAL_SRC_FILES_x86_64 += \ +libm_x86_64_arch_files += \ upstream-freebsd/lib/msun/src/e_sqrt.c \ upstream-freebsd/lib/msun/src/e_sqrtf.c \ upstream-freebsd/lib/msun/src/s_ceil.c \ upstream-freebsd/lib/msun/src/s_ceilf.c \ + upstream-freebsd/lib/msun/src/s_cos.c \ upstream-freebsd/lib/msun/src/s_fma.c \ upstream-freebsd/lib/msun/src/s_fmaf.c \ upstream-freebsd/lib/msun/src/s_floor.c \ @@ -336,6 +354,7 @@ LOCAL_SRC_FILES_x86_64 += \ upstream-freebsd/lib/msun/src/s_lrintf.c \ upstream-freebsd/lib/msun/src/s_rint.c \ upstream-freebsd/lib/msun/src/s_rintf.c \ + upstream-freebsd/lib/msun/src/s_sin.c \ upstream-freebsd/lib/msun/src/s_trunc.c \ upstream-freebsd/lib/msun/src/s_truncf.c \ @@ -344,6 +363,7 @@ LOCAL_SRC_FILES_x86_64 += \ libm_common_cflags := \ -DFLT_EVAL_METHOD=0 \ -std=c99 \ + -O3 \ -include $(LOCAL_PATH)/freebsd-compat.h \ -Wno-missing-braces \ -Wno-parentheses \ @@ -352,43 +372,13 @@ libm_common_cflags := \ -Wno-unknown-pragmas \ -fvisibility=hidden \ -LOCAL_ASFLAGS := \ - -Ibionic/libc \ - # Workaround the GCC "(long)fn -> lfn" optimization bug which will result in # self recursions for lrint, lrintf, and lrintl. # BUG: 14225968 libm_common_cflags += -fno-builtin-rint -fno-builtin-rintf -fno-builtin-rintl -libm_common_includes := $(LOCAL_PATH)/upstream-freebsd/lib/msun/src/ +libm_common_includes += $(LOCAL_PATH)/upstream-freebsd/lib/msun/src/ -libm_ld_includes := $(LOCAL_PATH)/upstream-freebsd/lib/msun/ld128/ - -ifeq ($(TARGET_USE_QCOM_BIONIC_OPTIMIZATION),true) - libm_arm_src_files += \ - arm/e_pow.S \ - arm/s_cos.S \ - arm/s_sin.S \ - arm/e_sqrtf.S \ - arm/e_sqrt.S - libm_arm_cflags += -DQCOM_NEON_OPTIMIZATION -fno-if-conversion - libm_arm_includes += $(LOCAL_PATH)/../libc/ - - libm_arm64_src_files += \ - arm64/e_pow64.S \ - upstream-freebsd/lib/msun/src/s_cos.c \ - upstream-freebsd/lib/msun/src/s_sin.c \ - upstream-freebsd/lib/msun/src/e_sqrtf.c \ - upstream-freebsd/lib/msun/src/e_sqrt.c - - libm_arm64_cflags += -DQCOM_NEON_OPTIMIZATION - libm_arm64_includes += $(LOCAL_PATH)/../libc/ -else - libm_common_src_files += \ - upstream-freebsd/lib/msun/src/s_cos.c \ - upstream-freebsd/lib/msun/src/s_sin.c \ - upstream-freebsd/lib/msun/src/e_sqrtf.c \ - upstream-freebsd/lib/msun/src/e_sqrt.c -endif +libm_ld_includes += $(LOCAL_PATH)/upstream-freebsd/lib/msun/ld128/ # # libm.a for target. @@ -402,25 +392,29 @@ LOCAL_C_INCLUDES += $(libm_common_includes) LOCAL_SRC_FILES := $(libm_common_src_files) LOCAL_SYSTEM_SHARED_LIBRARIES := libc +LOCAL_ASFLAGS := \ + -Ibionic/libc \ + # arch-specific settings -LOCAL_CFLAGS_arm := $(libm_arm_cflags) -LOCAL_C_INCLUDES_arm := $(LOCAL_PATH)/arm $(libm_arm_includes) -LOCAL_SRC_FILES_arm := arm/fenv.c $(libm_arm_src_files) +LOCAL_CFLAGS_arm := $(libm_arm_arch_cflags) +LOCAL_C_INCLUDES_arm := $(LOCAL_PATH)/arm $(libm_arm_arch_includes) +LOCAL_SRC_FILES_arm := arm/fenv.c $(libm_arm_arch_files) -LOCAL_CFLAGS_arm64 := $(libm_arm64_cflags) -LOCAL_C_INCLUDES_arm64 := $(libm_ld_includes) $(libm_arm64_includes) -LOCAL_SRC_FILES_arm64 := arm64/fenv.c $(libm_ld128_src_files) $(libm_arm64_src_files) +LOCAL_CFLAGS_arm64 := $(libm_arm64_arch_cflags) +LOCAL_C_INCLUDES_arm64 := $(libm_ld_includes) $(libm_arm64_arch_includes) +LOCAL_SRC_FILES_arm64 := arm64/fenv.c $(libm_ld128_src_files) $(libm_arm64_arch_files) LOCAL_C_INCLUDES_x86 := $(LOCAL_PATH)/i387 -LOCAL_SRC_FILES_x86 := i387/fenv.c +LOCAL_SRC_FILES_x86 := i387/fenv.c $(libm_x86_arch_files) LOCAL_C_INCLUDES_x86_64 := $(libm_ld_includes) -LOCAL_SRC_FILES_x86_64 := amd64/fenv.c $(libm_ld128_src_files) +LOCAL_SRC_FILES_x86_64 := amd64/fenv.c $(libm_ld128_src_files) $(libm_x86_64_arch_files) -LOCAL_SRC_FILES_mips := mips/fenv.c +LOCAL_SRC_FILES_mips += mips/fenv.c $(libm_mips_arch_files) LOCAL_C_INCLUDES_mips64 := $(libm_ld_includes) LOCAL_SRC_FILES_mips64 := mips/fenv.c $(libm_ld128_src_files) +LOCAL_SRC_FILES_mips64 += $(libm_mips_arch_files) include $(BUILD_STATIC_LIBRARY) diff --git a/libm/upstream-freebsd/lib/msun/src/e_pow.c b/libm/upstream-freebsd/lib/msun/src/e_pow.c index 4248f84ea..d54af9d68 100644 --- a/libm/upstream-freebsd/lib/msun/src/e_pow.c +++ b/libm/upstream-freebsd/lib/msun/src/e_pow.c @@ -94,11 +94,7 @@ ivln2_h = 1.44269502162933349609e+00, /* 0x3FF71547, 0x60000000 =24b 1/ln2*/ ivln2_l = 1.92596299112661746887e-08; /* 0x3E54AE0B, 0xF85DDF44 =1/ln2 tail*/ double -#if defined(KRAIT_NEON_OPTIMIZATION) || defined(QCOM_NEON_OPTIMIZATION) -__full_ieee754_pow(double x, double y) -#else __ieee754_pow(double x, double y) -#endif { double z,ax,z_h,z_l,p_h,p_l; double y1,t1,t2,r,s,t,u,v,w; @@ -110,35 +106,6 @@ __ieee754_pow(double x, double y) EXTRACT_WORDS(hy,ly,y); ix = hx&0x7fffffff; iy = hy&0x7fffffff; -#if defined(KRAIT_NEON_OPTIMIZATION) || defined(QCOM_NEON_OPTIMIZATION) - - if (ly == 0) { - if (hy == ly) { - /* y==0.0, x**0 = 1 */ - return one; - } - else if (iy > 0x7ff00000) { - /* y is NaN, return x+y (NaN) */ - return x+y; - } - } - else if (iy >= 0x7ff00000) { - /* y is NaN, return x+y (NaN) */ - return x+y; - } - - if (lx == 0) { - if (ix > 0x7ff00000) { - /* x is NaN, return x+y (NaN) */ - return x+y; - } - } - else if (ix >= 0x7ff00000) { - /* x is NaN, return x+y (NaN) */ - return x+y; - } - -#else /* y==zero: x**0 = 1 */ if((iy|ly)==0) return one; @@ -149,7 +116,7 @@ __ieee754_pow(double x, double y) if(ix > 0x7ff00000 || ((ix==0x7ff00000)&&(lx!=0)) || iy > 0x7ff00000 || ((iy==0x7ff00000)&&(ly!=0))) return (x+0.0)+(y+0.0); -#endif + /* determine if y is an odd int when x < 0 * yisint = 0 ... y is not an integer * yisint = 1 ... y is an odd int diff --git a/libm/upstream-freebsd/lib/msun/src/k_cos.c b/libm/upstream-freebsd/lib/msun/src/k_cos.c index 6037e0dc0..a0baaacd7 100644 --- a/libm/upstream-freebsd/lib/msun/src/k_cos.c +++ b/libm/upstream-freebsd/lib/msun/src/k_cos.c @@ -68,7 +68,7 @@ C6 = -1.13596475577881948265e-11; /* 0xBDA8FAE9, 0xBE8838D4 */ double __kernel_cos(double x, double y) { -#if defined(KRAIT_NEON_OPTIMIZATION) || defined(QCOM_NEON_OPTIMIZATION) +#if defined(QCOM_NEON_OPTIMIZATION) double hz,z,zz,r,w,k; z = x*x; diff --git a/libm/upstream-freebsd/lib/msun/src/k_sin.c b/libm/upstream-freebsd/lib/msun/src/k_sin.c index afd2da897..25b2c5c08 100644 --- a/libm/upstream-freebsd/lib/msun/src/k_sin.c +++ b/libm/upstream-freebsd/lib/msun/src/k_sin.c @@ -59,7 +59,7 @@ S6 = 1.58969099521155010221e-10; /* 0x3DE5D93A, 0x5ACFD57C */ double __kernel_sin(double x, double y, int iy) { -#if defined(KRAIT_NEON_OPTIMIZATION) || defined(QCOM_NEON_OPTIMIZATION) +#if defined(QCOM_NEON_OPTIMIZATION) double z,zz,r,v; z = x*x; diff --git a/libm/upstream-freebsd/lib/msun/src/math_private.h b/libm/upstream-freebsd/lib/msun/src/math_private.h index 77520d778..a8988d0c7 100644 --- a/libm/upstream-freebsd/lib/msun/src/math_private.h +++ b/libm/upstream-freebsd/lib/msun/src/math_private.h @@ -723,9 +723,9 @@ irintl(long double x) #define __ieee754_remainderf remainderf #define __ieee754_scalbf scalbf -#if defined(KRAIT_NEON_OPTIMIZATION) || defined(QCOM_NEON_OPTIMIZATION) +#if defined(QCOM_NEON_OPTIMIZATION) && defined(__ARM_NEON__) int __kernel_rem_pio2(double*,double*,int,int,int) __attribute__((pcs("aapcs-vfp"))); -double __full_ieee754_pow(double,double); +//double __full_ieee754_pow(double,double); #ifndef INLINE_REM_PIO2 int __ieee754_rem_pio2(double,double*) __attribute__((pcs("aapcs-vfp"))); #endif From a2cd3dd0ffbc61f6f578ec1777090db5374c28ea Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Thu, 23 Jul 2015 20:27:42 -0700 Subject: [PATCH 76/89] Add optimized cortex-a7/cortex-a53 memset/memcpy. This adds an optimized memset that is ~20% faster for cortex-a7 and cortex-a53. In addition, add a 32 bit optimized cortex-a53 memcpy. Since __str{cat,cpy}_chk.S use memcpy_base.S, add a 32 bit cortex-a53 copy of this code too. Bug: 22696180 Change-Id: Iec92a3f82d51243cca76c9aff9f35d920ff865ae --- .../arch-arm/cortex-a53/bionic/__strcat_chk.S | 214 ++++++++++++++++++ .../arch-arm/cortex-a53/bionic/__strcpy_chk.S | 179 +++++++++++++++ libc/arch-arm/cortex-a53/bionic/memcpy.S | 102 +++++++++ libc/arch-arm/cortex-a53/bionic/memcpy_base.S | 148 ++++++++++++ libc/arch-arm/cortex-a53/cortex-a53.mk | 16 +- libc/arch-arm/cortex-a7/bionic/memset.S | 180 +++++++++++++++ libc/arch-arm/cortex-a7/cortex-a7.mk | 19 +- 7 files changed, 852 insertions(+), 6 deletions(-) create mode 100644 libc/arch-arm/cortex-a53/bionic/__strcat_chk.S create mode 100644 libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S create mode 100644 libc/arch-arm/cortex-a53/bionic/memcpy.S create mode 100644 libc/arch-arm/cortex-a53/bionic/memcpy_base.S create mode 100644 libc/arch-arm/cortex-a7/bionic/memset.S diff --git a/libc/arch-arm/cortex-a53/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a53/bionic/__strcat_chk.S new file mode 100644 index 000000000..476e2aece --- /dev/null +++ b/libc/arch-arm/cortex-a53/bionic/__strcat_chk.S @@ -0,0 +1,214 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + + .syntax unified + + .thumb + .thumb_func + +// Get the length of src string, then get the source of the dst string. +// Check that the two lengths together don't exceed the threshold, then +// do a memcpy of the data. +ENTRY(__strcat_chk) + pld [r0, #0] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + push {r4, r5} + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 + + mov lr, r2 + + // Save the dst register to r5 + mov r5, r0 + + // Zero out r4 + eor r4, r4, r4 + + // r1 contains the address of the string to count. +.L_strlen_start: + mov r0, r1 + ands r3, r1, #7 + beq .L_mainloop + + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq .L_align_to_32 + + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_32: + bcc .L_align_to_64 + ands ip, r3, #2 + beq .L_align_to_64 + + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_64: + tst r3, #4 + beq .L_mainloop + ldr r3, [r1], #4 + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + + .p2align 2 +.L_mainloop: + ldrd r2, r3, [r1], #8 + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne .L_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + b .L_mainloop + +.L_update_count_and_finish: + sub r3, r1, r0 + sub r3, r3, #1 + b .L_finish + +.L_zero_in_first_register: + sub r3, r1, r0 + lsls r2, ip, #17 + bne .L_sub8_and_finish + bcs .L_sub7_and_finish + lsls ip, ip, #1 + bne .L_sub6_and_finish + + sub r3, r3, #5 + b .L_finish + +.L_sub8_and_finish: + sub r3, r3, #8 + b .L_finish + +.L_sub7_and_finish: + sub r3, r3, #7 + b .L_finish + +.L_sub6_and_finish: + sub r3, r3, #6 + b .L_finish + +.L_zero_in_second_register: + sub r3, r1, r0 + lsls r2, ip, #17 + bne .L_sub4_and_finish + bcs .L_sub3_and_finish + lsls ip, ip, #1 + bne .L_sub2_and_finish + + sub r3, r3, #1 + b .L_finish + +.L_sub4_and_finish: + sub r3, r3, #4 + b .L_finish + +.L_sub3_and_finish: + sub r3, r3, #3 + b .L_finish + +.L_sub2_and_finish: + sub r3, r3, #2 + +.L_finish: + cmp r4, #0 + bne .L_strlen_done + + // Time to get the dst string length. + mov r1, r5 + + // Save the original source address to r5. + mov r5, r0 + + // Save the current length (adding 1 for the terminator). + add r4, r3, #1 + b .L_strlen_start + + // r0 holds the pointer to the dst string. + // r3 holds the dst string length. + // r4 holds the src string length + 1. +.L_strlen_done: + add r2, r3, r4 + cmp r2, lr + bhi __strcat_chk_failed + + // Set up the registers for the memcpy code. + mov r1, r5 + pld [r1, #64] + mov r2, r4 + add r0, r0, r3 + pop {r4, r5} +END(__strcat_chk) + +#define MEMCPY_BASE __strcat_chk_memcpy_base +#include "memcpy_base.S" + +ENTRY_PRIVATE(__strcat_chk_failed) + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_STRCAT_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+4) +END(__strcat_chk_failed) + + .data +error_string: + .string "strcat: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S new file mode 100644 index 000000000..6078a51b8 --- /dev/null +++ b/libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S @@ -0,0 +1,179 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + + .syntax unified + + .thumb + .thumb_func + +// Get the length of the source string first, then do a memcpy of the data +// instead of a strcpy. +ENTRY(__strcpy_chk) + pld [r0, #0] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + + mov lr, r2 + mov r0, r1 + + ands r3, r1, #7 + beq .L_mainloop + + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq .L_align_to_32 + + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_32: + bcc .L_align_to_64 + ands ip, r3, #2 + beq .L_align_to_64 + + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_64: + tst r3, #4 + beq .L_mainloop + ldr r3, [r0], #4 + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + + .p2align 2 +.L_mainloop: + ldrd r2, r3, [r0], #8 + + pld [r0, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne .L_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + b .L_mainloop + +.L_update_count_and_finish: + sub r3, r0, r1 + sub r3, r3, #1 + b .L_check_size + +.L_zero_in_first_register: + sub r3, r0, r1 + lsls r2, ip, #17 + bne .L_sub8_and_finish + bcs .L_sub7_and_finish + lsls ip, ip, #1 + bne .L_sub6_and_finish + + sub r3, r3, #5 + b .L_check_size + +.L_sub8_and_finish: + sub r3, r3, #8 + b .L_check_size + +.L_sub7_and_finish: + sub r3, r3, #7 + b .L_check_size + +.L_sub6_and_finish: + sub r3, r3, #6 + b .L_check_size + +.L_zero_in_second_register: + sub r3, r0, r1 + lsls r2, ip, #17 + bne .L_sub4_and_finish + bcs .L_sub3_and_finish + lsls ip, ip, #1 + bne .L_sub2_and_finish + + sub r3, r3, #1 + b .L_check_size + +.L_sub4_and_finish: + sub r3, r3, #4 + b .L_check_size + +.L_sub3_and_finish: + sub r3, r3, #3 + b .L_check_size + +.L_sub2_and_finish: + sub r3, r3, #2 + +.L_check_size: + pld [r1, #0] + pld [r1, #64] + ldr r0, [sp] + cmp r3, lr + bhs __strcpy_chk_failed + + // Add 1 for copy length to get the string terminator. + add r2, r3, #1 +END(__strcpy_chk) + +#define MEMCPY_BASE __strcpy_chk_memcpy_base +#include "memcpy_base.S" + +ENTRY_PRIVATE(__strcpy_chk_failed) + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+4) +END(__strcpy_chk_failed) + + .data +error_string: + .string "strcpy: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a53/bionic/memcpy.S b/libc/arch-arm/cortex-a53/bionic/memcpy.S new file mode 100644 index 000000000..1b29543e7 --- /dev/null +++ b/libc/arch-arm/cortex-a53/bionic/memcpy.S @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// Prototype: void *memcpy (void *dst, const void *src, size_t count). + +#include +#include + + .text + .syntax unified + .fpu neon + +ENTRY(__memcpy_chk) + cmp r2, r3 + bhi __memcpy_chk_fail + + // Fall through to memcpy... +END(__memcpy_chk) + +ENTRY(memcpy) + pld [r1, #64] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 +END(memcpy) + +#define MEMCPY_BASE __memcpy_base +#include "memcpy_base.S" + +ENTRY_PRIVATE(__memcpy_chk_fail) + // Preserve lr for backtrace. + push {lr} + .cfi_def_cfa_offset 4 + .cfi_rel_offset lr, 0 + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) +END(__memcpy_chk_fail) + + .data +error_string: + .string "memcpy: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a53/bionic/memcpy_base.S b/libc/arch-arm/cortex-a53/bionic/memcpy_base.S new file mode 100644 index 000000000..b54f141cc --- /dev/null +++ b/libc/arch-arm/cortex-a53/bionic/memcpy_base.S @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +ENTRY_PRIVATE(MEMCPY_BASE) + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + + // Assumes that n >= 0, and dst, src are valid pointers. + cmp r2, #16 + blo .L_copy_less_than_16_unknown_align + +.L_copy_unknown_alignment: + // Unknown alignment of src and dst. + // Assumes that the first few bytes have already been prefetched. + + // Align destination to 128 bits. The mainloop store instructions + // require this alignment or they will throw an exception. + rsb r3, r0, #0 + ands r3, r3, #0xF + beq 2f + + // Copy up to 15 bytes (count in r3). + sub r2, r2, r3 + movs ip, r3, lsl #31 + + itt mi + ldrbmi lr, [r1], #1 + strbmi lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1], #1 + strbcs ip, [r0], #1 + strbcs lr, [r0], #1 + + movs ip, r3, lsl #29 + bge 1f + // Copies 4 bytes, dst 32 bits aligned before, at least 64 bits after. + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! +1: bcc 2f + // Copies 8 bytes, dst 64 bits aligned before, at least 128 bits after. + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0, :64]! + +2: // Make sure we have at least 64 bytes to copy. + subs r2, r2, #64 + blo 2f + +1: // The main loop copies 64 bytes at a time. + vld1.8 {d0 - d3}, [r1]! + vld1.8 {d4 - d7}, [r1]! + subs r2, r2, #64 + vstmia r0!, {d0 - d7} + pld [r1, #(64*10)] + bhs 1b + +2: // Fix-up the remaining count and make sure we have >= 32 bytes left. + adds r2, r2, #32 + blo 3f + + // 32 bytes. These cache lines were already preloaded. + vld1.8 {d0 - d3}, [r1]! + sub r2, r2, #32 + vst1.8 {d0 - d3}, [r0, :128]! +3: // Less than 32 left. + add r2, r2, #32 + tst r2, #0x10 + beq .L_copy_less_than_16_unknown_align + // Copies 16 bytes, destination 128 bits aligned. + vld1.8 {d0, d1}, [r1]! + vst1.8 {d0, d1}, [r0, :128]! + +.L_copy_less_than_16_unknown_align: + // Copy up to 15 bytes (count in r2). + movs ip, r2, lsl #29 + bcc 1f + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0]! +1: bge 2f + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! + +2: // Copy 0 to 4 bytes. + lsls r2, r2, #31 + itt ne + ldrbne lr, [r1], #1 + strbne lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1] + strbcs ip, [r0], #1 + strbcs lr, [r0] + + pop {r0, pc} +END(MEMCPY_BASE) diff --git a/libc/arch-arm/cortex-a53/cortex-a53.mk b/libc/arch-arm/cortex-a53/cortex-a53.mk index e6fc66d6e..3f75ebbfc 100644 --- a/libc/arch-arm/cortex-a53/cortex-a53.mk +++ b/libc/arch-arm/cortex-a53/cortex-a53.mk @@ -1,12 +1,18 @@ libc_bionic_src_files_arm += \ - arch-arm/cortex-a15/bionic/memchr.S \ - arch-arm/cortex-a15/bionic/memcpy.S \ - arch-arm/cortex-a15/bionic/memset.S \ + arch-arm/cortex-a53/bionic/memcpy.S \ + arch-arm/cortex-a53/bionic/__strcat_chk.S \ + arch-arm/cortex-a53/bionic/__strcpy_chk.S \ + +libc_bionic_src_files_arm += \ + arch-arm/cortex-a7/bionic/memset.S \ + +libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/strcat.S \ arch-arm/cortex-a15/bionic/strcmp.S \ arch-arm/cortex-a15/bionic/strcpy.S \ arch-arm/cortex-a15/bionic/strlen.S \ arch-arm/cortex-a15/bionic/stpcpy.S \ - arch-arm/cortex-a15/bionic/__strcat_chk.S \ - arch-arm/cortex-a15/bionic/__strcpy_chk.S \ arch-arm/krait/bionic/memmove.S + +libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memchr.S diff --git a/libc/arch-arm/cortex-a7/bionic/memset.S b/libc/arch-arm/cortex-a7/bionic/memset.S new file mode 100644 index 000000000..6365b066a --- /dev/null +++ b/libc/arch-arm/cortex-a7/bionic/memset.S @@ -0,0 +1,180 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include + + /* + * Optimized memset() for ARM. + * + * memset() returns its first argument. + */ + + .fpu neon + .syntax unified + +ENTRY(__memset_chk) + cmp r2, r3 + bls .L_done + + // Preserve lr for backtrace. + push {lr} + .cfi_def_cfa_offset 4 + .cfi_rel_offset lr, 0 + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMSET_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) +END(__memset_chk) + +ENTRY(bzero) + mov r2, r1 + mov r1, #0 +.L_done: + // Fall through to memset... +END(bzero) + +ENTRY(memset) + mov r3, r0 + // At this point only d0, d1 are going to be used below. + vdup.8 q0, r1 + cmp r2, #16 + blo .L_set_less_than_16_unknown_align + +.L_check_alignment: + // Align destination to a double word to avoid the store crossing + // a cache line boundary. + ands ip, r3, #7 + bne .L_do_double_word_align + +.L_double_word_aligned: + // Duplicate since the less than 64 can use d2, d3. + vmov q1, q0 + subs r2, #64 + blo .L_set_less_than_64 + + // Duplicate the copy value so that we can store 64 bytes at a time. + vmov q2, q0 + vmov q3, q0 + +1: // Main loop stores 64 bytes at a time. + subs r2, #64 + vstmia r3!, {d0 - d7} + bge 1b + +.L_set_less_than_64: + // Restore r2 to the count of bytes left to set. + add r2, #64 + lsls ip, r2, #27 + bcc .L_set_less_than_32 + // Set 32 bytes. + vstmia r3!, {d0 - d3} + +.L_set_less_than_32: + bpl .L_set_less_than_16 + // Set 16 bytes. + vstmia r3!, {d0, d1} + +.L_set_less_than_16: + // Less than 16 bytes to set. + lsls ip, r2, #29 + bcc .L_set_less_than_8 + + // Set 8 bytes. + vstmia r3!, {d0} + +.L_set_less_than_8: + bpl .L_set_less_than_4 + // Set 4 bytes + vst1.32 {d0[0]}, [r3]! + +.L_set_less_than_4: + lsls ip, r2, #31 + it ne + strbne r1, [r3], #1 + itt cs + strbcs r1, [r3], #1 + strbcs r1, [r3] + bx lr + +.L_do_double_word_align: + rsb ip, ip, #8 + sub r2, r2, ip + + // Do this comparison now, otherwise we'll need to save a + // register to the stack since we've used all available + // registers. + cmp ip, #4 + blo 1f + + // Need to do a four byte copy. + movs ip, ip, lsl #31 + it mi + strbmi r1, [r3], #1 + itt cs + strbcs r1, [r3], #1 + strbcs r1, [r3], #1 + vst1.32 {d0[0]}, [r3]! + b .L_double_word_aligned + +1: + // No four byte copy. + movs ip, ip, lsl #31 + it mi + strbmi r1, [r3], #1 + itt cs + strbcs r1, [r3], #1 + strbcs r1, [r3], #1 + b .L_double_word_aligned + +.L_set_less_than_16_unknown_align: + // Set up to 15 bytes. + movs ip, r2, lsl #29 + bcc 1f + vst1.8 {d0}, [r3]! +1: bge 2f + vst1.32 {d0[0]}, [r3]! +2: movs ip, r2, lsl #31 + it mi + strbmi r1, [r3], #1 + itt cs + strbcs r1, [r3], #1 + strbcs r1, [r3], #1 + bx lr +END(memset) + + .data +error_string: + .string "memset: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a7/cortex-a7.mk b/libc/arch-arm/cortex-a7/cortex-a7.mk index 9af03d94b..b6af4dadb 100644 --- a/libc/arch-arm/cortex-a7/cortex-a7.mk +++ b/libc/arch-arm/cortex-a7/cortex-a7.mk @@ -1 +1,18 @@ -include bionic/libc/arch-arm/cortex-a15/cortex-a15.mk +libc_bionic_src_files_arm += \ + arch-arm/cortex-a7/bionic/memset.S \ + +libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memcpy.S \ + arch-arm/cortex-a15/bionic/stpcpy.S \ + arch-arm/cortex-a15/bionic/strcat.S \ + arch-arm/cortex-a15/bionic/__strcat_chk.S \ + arch-arm/cortex-a15/bionic/strcmp.S \ + arch-arm/cortex-a15/bionic/strcpy.S \ + arch-arm/cortex-a15/bionic/__strcpy_chk.S \ + arch-arm/cortex-a15/bionic/strlen.S \ + +libc_bionic_src_files_arm += \ + arch-arm/generic/bionic/memcmp.S \ + +libc_bionic_src_files_arm += \ + arch-arm/denver/bionic/memmove.S \ From 70533720eac28474835925b5cd26a2ca08438dee Mon Sep 17 00:00:00 2001 From: LuK1337 Date: Sun, 9 Aug 2015 19:12:31 +0200 Subject: [PATCH 77/89] cortex-a7: set proper memcmp.S path * There is no memcmp.S in the arch-arm/generic/bionic/ folder Change-Id: I069224c9f43e3de516005a93551729cc620c3bdc --- libc/arch-arm/cortex-a7/cortex-a7.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/arch-arm/cortex-a7/cortex-a7.mk b/libc/arch-arm/cortex-a7/cortex-a7.mk index b6af4dadb..c946c2cb1 100644 --- a/libc/arch-arm/cortex-a7/cortex-a7.mk +++ b/libc/arch-arm/cortex-a7/cortex-a7.mk @@ -12,7 +12,7 @@ libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/strlen.S \ libc_bionic_src_files_arm += \ - arch-arm/generic/bionic/memcmp.S \ + arch-arm/bionic/memcmp.S \ libc_bionic_src_files_arm += \ arch-arm/denver/bionic/memmove.S \ From d937421274347e883e4092d1e7da970bb667eb8e Mon Sep 17 00:00:00 2001 From: LuK1337 Date: Mon, 10 Aug 2015 23:14:53 +0200 Subject: [PATCH 78/89] cortex-a7: Fix build * stpcpy.S is already defined * memchr.S was missing Change-Id: Ia7b21373da6dc0bbc2c3f3b1a46050946af3d31b --- libc/arch-arm/cortex-a7/cortex-a7.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/arch-arm/cortex-a7/cortex-a7.mk b/libc/arch-arm/cortex-a7/cortex-a7.mk index c946c2cb1..646c8b612 100644 --- a/libc/arch-arm/cortex-a7/cortex-a7.mk +++ b/libc/arch-arm/cortex-a7/cortex-a7.mk @@ -2,8 +2,8 @@ libc_bionic_src_files_arm += \ arch-arm/cortex-a7/bionic/memset.S \ libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memchr.S \ arch-arm/cortex-a15/bionic/memcpy.S \ - arch-arm/cortex-a15/bionic/stpcpy.S \ arch-arm/cortex-a15/bionic/strcat.S \ arch-arm/cortex-a15/bionic/__strcat_chk.S \ arch-arm/cortex-a15/bionic/strcmp.S \ From 4bb237a9c05251c924edf31920a0e6ebb87547a0 Mon Sep 17 00:00:00 2001 From: Kyle Repinski Date: Wed, 15 Apr 2015 23:14:36 -0500 Subject: [PATCH 79/89] libc: Fix strict-aliasing violations. Change-Id: I4da70ca9eb4e7ebd997e99a8d25448bf110645b4 --- libc/bionic/mbstate.cpp | 4 ++-- libc/bionic/wchar.cpp | 2 +- libc/dns/net/getaddrinfo.c | 2 +- libc/include/netinet/in6.h | 36 ++++++++++++++++++------------------ libc/include/wchar.h | 5 ++++- 5 files changed, 26 insertions(+), 23 deletions(-) diff --git a/libc/bionic/mbstate.cpp b/libc/bionic/mbstate.cpp index cb327d839..35ae76a3c 100644 --- a/libc/bionic/mbstate.cpp +++ b/libc/bionic/mbstate.cpp @@ -47,11 +47,11 @@ __LIBC_HIDDEN__ uint8_t mbstate_get_byte(const mbstate_t* ps, int n) { __LIBC_HIDDEN__ size_t reset_and_return_illegal(int _errno, mbstate_t* ps) { errno = _errno; - *(reinterpret_cast(ps->__seq)) = 0; + ps->__seq32 = 0; return __MB_ERR_ILLEGAL_SEQUENCE; } __LIBC_HIDDEN__ size_t reset_and_return(int _return, mbstate_t* ps) { - *(reinterpret_cast(ps->__seq)) = 0; + ps->__seq32 = 0; return _return; } diff --git a/libc/bionic/wchar.cpp b/libc/bionic/wchar.cpp index e0879b9df..723639cb8 100644 --- a/libc/bionic/wchar.cpp +++ b/libc/bionic/wchar.cpp @@ -54,7 +54,7 @@ // int mbsinit(const mbstate_t* ps) { - return (ps == NULL || (*(reinterpret_cast(ps->__seq)) == 0)); + return (ps == NULL || ps->__seq32 == 0); } size_t mbrtowc(wchar_t* pwc, const char* s, size_t n, mbstate_t* ps) { diff --git a/libc/dns/net/getaddrinfo.c b/libc/dns/net/getaddrinfo.c index 2612d6a4d..ea1b5695f 100644 --- a/libc/dns/net/getaddrinfo.c +++ b/libc/dns/net/getaddrinfo.c @@ -1544,7 +1544,7 @@ _get_scope(const struct sockaddr *addr) /* RFC 4380, section 2.6 */ #define IN6_IS_ADDR_TEREDO(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == ntohl(0x20010000))) + (((a)->s6_addr32[0]) == ntohl(0x20010000)) /* RFC 3056, section 2. */ #define IN6_IS_ADDR_6TO4(a) \ diff --git a/libc/include/netinet/in6.h b/libc/include/netinet/in6.h index 7f3286ae9..f33e41ff5 100644 --- a/libc/include/netinet/in6.h +++ b/libc/include/netinet/in6.h @@ -31,28 +31,28 @@ #include #define IN6_IS_ADDR_UNSPECIFIED(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[4]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[8]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[12]) == 0)) + (((a)->s6_addr32[0] == 0) && \ + ((a)->s6_addr32[1] == 0) && \ + ((a)->s6_addr32[2] == 0) && \ + ((a)->s6_addr32[3] == 0)) #define IN6_IS_ADDR_LOOPBACK(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[4]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[8]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[12]) == ntohl(1))) + (((a)->s6_addr32[0] == 0) && \ + ((a)->s6_addr32[1] == 0) && \ + ((a)->s6_addr32[2] == 0) && \ + ((a)->s6_addr32[3] == ntohl(1))) #define IN6_IS_ADDR_V4COMPAT(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[4]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[8]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[12]) != 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[12]) != ntohl(1))) + (((a)->s6_addr32[0] == 0) && \ + ((a)->s6_addr32[1] == 0) && \ + ((a)->s6_addr32[2] == 0) && \ + ((a)->s6_addr32[3] != 0) && \ + ((a)->s6_addr32[3] != ntohl(1))) -#define IN6_IS_ADDR_V4MAPPED(a) \ - ((*(const uint32_t *)(const void *)(&(a)->s6_addr[0]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[4]) == 0) && \ - (*(const uint32_t *)(const void *)(&(a)->s6_addr[8]) == ntohl(0x0000ffff))) +#define IN6_IS_ADDR_V4MAPPED(a) \ + (((a)->s6_addr32[0] == 0) && \ + ((a)->s6_addr32[1] == 0) && \ + ((a)->s6_addr32[2] == ntohl(0x0000ffff))) #define IN6_IS_ADDR_LINKLOCAL(a) \ (((a)->s6_addr[0] == 0xfe) && (((a)->s6_addr[1] & 0xc0) == 0x80)) @@ -65,7 +65,7 @@ (((a)->s6_addr[0] & 0xfe) == 0xfc) #define IN6_IS_ADDR_MULTICAST(a) \ - (((__const uint8_t *) (a))[0] == 0xff) + ((a)->s6_addr[0] == 0xff) #define IPV6_ADDR_SCOPE_NODELOCAL 0x01 diff --git a/libc/include/wchar.h b/libc/include/wchar.h index e0e5c82fe..dc328889f 100644 --- a/libc/include/wchar.h +++ b/libc/include/wchar.h @@ -41,7 +41,10 @@ __BEGIN_DECLS typedef __WINT_TYPE__ wint_t; typedef struct { - uint8_t __seq[4]; + union { + uint8_t __seq[4]; + uint32_t __seq32; + }; #ifdef __LP64__ char __reserved[4]; #endif From 94c894ba917af45baafbb8c24537013280b95f5f Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Thu, 18 Sep 2014 11:23:58 -0700 Subject: [PATCH 80/89] Fix lgamma_r/lgammaf_r/lgammal_r for -0. Upstream has implemented lgammal/lgammal_r for ld128, and fixed the sign problem we reported with all the lgamma*_r functions and -0. Bug: 17471883 Change-Id: Ibb175d9cab67efae75f1010796fd44c9ba6ce4fc --- libm/Android.mk | 2 + libm/include/math.h | 5 +- .../lib/msun/ld128/e_lgammal_r.c | 333 ++++++++++++++++++ libm/upstream-freebsd/lib/msun/src/e_lgamma.c | 6 + .../lib/msun/src/e_lgamma_r.c | 25 +- .../lib/msun/src/e_lgammaf_r.c | 177 +++++----- .../upstream-freebsd/lib/msun/src/e_lgammal.c | 25 ++ .../upstream-freebsd/lib/msun/src/imprecise.c | 1 - tests/math_test.cpp | 39 ++ 9 files changed, 511 insertions(+), 102 deletions(-) create mode 100644 libm/upstream-freebsd/lib/msun/ld128/e_lgammal_r.c create mode 100644 libm/upstream-freebsd/lib/msun/src/e_lgammal.c diff --git a/libm/Android.mk b/libm/Android.mk index 40098b720..7932aea2a 100644 --- a/libm/Android.mk +++ b/libm/Android.mk @@ -178,6 +178,7 @@ libm_ld128_src_files = \ upstream-freebsd/lib/msun/src/e_atanhl.c \ upstream-freebsd/lib/msun/src/e_fmodl.c \ upstream-freebsd/lib/msun/src/e_hypotl.c \ + upstream-freebsd/lib/msun/src/e_lgammal.c \ upstream-freebsd/lib/msun/src/e_remainderl.c \ upstream-freebsd/lib/msun/src/e_sqrtl.c \ upstream-freebsd/lib/msun/src/s_asinhl.c \ @@ -215,6 +216,7 @@ libm_ld128_src_files = \ libm_ld128_src_files += \ upstream-freebsd/lib/msun/ld128/invtrig.c \ + upstream-freebsd/lib/msun/ld128/e_lgammal_r.c \ upstream-freebsd/lib/msun/ld128/k_cosl.c \ upstream-freebsd/lib/msun/ld128/k_sinl.c \ upstream-freebsd/lib/msun/ld128/k_tanl.c \ diff --git a/libm/include/math.h b/libm/include/math.h index 3eca1407d..dec6389f6 100644 --- a/libm/include/math.h +++ b/libm/include/math.h @@ -459,9 +459,12 @@ long double tanhl(long double); long double tanl(long double); long double tgammal(long double); long double truncl(long double); - #endif /* __ISO_C_VISIBLE >= 1999 */ +#if __BSD_VISIBLE +long double lgammal_r(long double, int *); +#endif + #if defined(_GNU_SOURCE) void sincos(double, double*, double*); void sincosf(float, float*, float*); diff --git a/libm/upstream-freebsd/lib/msun/ld128/e_lgammal_r.c b/libm/upstream-freebsd/lib/msun/ld128/e_lgammal_r.c new file mode 100644 index 000000000..13f8f75a4 --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/ld128/e_lgammal_r.c @@ -0,0 +1,333 @@ +/* @(#)e_lgamma_r.c 1.3 95/01/18 */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +#include +__FBSDID("$FreeBSD$"); + +/* + * See e_lgamma_r.c for complete comments. + * + * Converted to long double by Steven G. Kargl. + */ + +#include "fpmath.h" +#include "math.h" +#include "math_private.h" + +static const volatile double vzero = 0; + +static const double +zero= 0, +half= 0.5, +one = 1; + +static const long double +pi = 3.14159265358979323846264338327950288e+00L; +/* + * Domain y in [0x1p-119, 0.28], range ~[-1.4065e-36, 1.4065e-36]: + * |(lgamma(2 - y) + y / 2) / y - a(y)| < 2**-119.1 + */ +static const long double +a0 = 7.72156649015328606065120900824024296e-02L, +a1 = 3.22467033424113218236207583323018498e-01L, +a2 = 6.73523010531980951332460538330282217e-02L, +a3 = 2.05808084277845478790009252803463129e-02L, +a4 = 7.38555102867398526627292839296001626e-03L, +a5 = 2.89051033074152328576829509522483468e-03L, +a6 = 1.19275391170326097618357349881842913e-03L, +a7 = 5.09669524743042462515256340206203019e-04L, +a8 = 2.23154758453578096143609255559576017e-04L, +a9 = 9.94575127818397632126978731542755129e-05L, +a10 = 4.49262367375420471287545895027098145e-05L, +a11 = 2.05072127845117995426519671481628849e-05L, +a12 = 9.43948816959096748454087141447939513e-06L, +a13 = 4.37486780697359330303852050718287419e-06L, +a14 = 2.03920783892362558276037363847651809e-06L, +a15 = 9.55191070057967287877923073200324649e-07L, +a16 = 4.48993286185740853170657139487620560e-07L, +a17 = 2.13107543597620911675316728179563522e-07L, +a18 = 9.70745379855304499867546549551023473e-08L, +a19 = 5.61889970390290257926487734695402075e-08L, +a20 = 6.42739653024130071866684358960960951e-09L, +a21 = 3.34491062143649291746195612991870119e-08L, +a22 = -1.57068547394315223934653011440641472e-08L, +a23 = 1.30812825422415841213733487745200632e-08L; +/* + * Domain x in [tc-0.24, tc+0.28], range ~[-6.3201e-37, 6.3201e-37]: + * |(lgamma(x) - tf) - t(x - tc)| < 2**-120.3. + */ +static const long double +tc = 1.46163214496836234126265954232572133e+00L, +tf = -1.21486290535849608095514557177691584e-01L, +tt = 1.57061739945077675484237837992951704e-36L, +t0 = -1.99238329499314692728655623767019240e-36L, +t1 = -6.08453430711711404116887457663281416e-35L, +t2 = 4.83836122723810585213722380854828904e-01L, +t3 = -1.47587722994530702030955093950668275e-01L, +t4 = 6.46249402389127526561003464202671923e-02L, +t5 = -3.27885410884813055008502586863748063e-02L, +t6 = 1.79706751152103942928638276067164935e-02L, +t7 = -1.03142230366363872751602029672767978e-02L, +t8 = 6.10053602051788840313573150785080958e-03L, +t9 = -3.68456960831637325470641021892968954e-03L, +t10 = 2.25976482322181046611440855340968560e-03L, +t11 = -1.40225144590445082933490395950664961e-03L, +t12 = 8.78232634717681264035014878172485575e-04L, +t13 = -5.54194952796682301220684760591403899e-04L, +t14 = 3.51912956837848209220421213975000298e-04L, +t15 = -2.24653443695947456542669289367055542e-04L, +t16 = 1.44070395420840737695611929680511823e-04L, +t17 = -9.27609865550394140067059487518862512e-05L, +t18 = 5.99347334438437081412945428365433073e-05L, +t19 = -3.88458388854572825603964274134801009e-05L, +t20 = 2.52476631610328129217896436186551043e-05L, +t21 = -1.64508584981658692556994212457518536e-05L, +t22 = 1.07434583475987007495523340296173839e-05L, +t23 = -7.03070407519397260929482550448878399e-06L, +t24 = 4.60968590693753579648385629003100469e-06L, +t25 = -3.02765473778832036018438676945512661e-06L, +t26 = 1.99238771545503819972741288511303401e-06L, +t27 = -1.31281299822614084861868817951788579e-06L, +t28 = 8.60844432267399655055574642052370223e-07L, +t29 = -5.64535486432397413273248363550536374e-07L, +t30 = 3.99357783676275660934903139592727737e-07L, +t31 = -2.95849029193433121795495215869311610e-07L, +t32 = 1.37790144435073124976696250804940384e-07L; +/* + * Domain y in [-0.1, 0.232], range ~[-1.4046e-37, 1.4181e-37]: + * |(lgamma(1 + y) + 0.5 * y) / y - u(y) / v(y)| < 2**-122.8 + */ +static const long double +u0 = -7.72156649015328606065120900824024311e-02L, +u1 = 4.24082772271938167430983113242482656e-01L, +u2 = 2.96194003481457101058321977413332171e+00L, +u3 = 6.49503267711258043997790983071543710e+00L, +u4 = 7.40090051288150177152835698948644483e+00L, +u5 = 4.94698036296756044610805900340723464e+00L, +u6 = 2.00194224610796294762469550684947768e+00L, +u7 = 4.82073087750608895996915051568834949e-01L, +u8 = 6.46694052280506568192333848437585427e-02L, +u9 = 4.17685526755100259316625348933108810e-03L, +u10 = 9.06361003550314327144119307810053410e-05L, +v1 = 5.15937098592887275994320496999951947e+00L, +v2 = 1.14068418766251486777604403304717558e+01L, +v3 = 1.41164839437524744055723871839748489e+01L, +v4 = 1.07170702656179582805791063277960532e+01L, +v5 = 5.14448694179047879915042998453632434e+00L, +v6 = 1.55210088094585540637493826431170289e+00L, +v7 = 2.82975732849424562719893657416365673e-01L, +v8 = 2.86424622754753198010525786005443539e-02L, +v9 = 1.35364253570403771005922441442688978e-03L, +v10 = 1.91514173702398375346658943749580666e-05L, +v11 = -3.25364686890242327944584691466034268e-08L; +/* + * Domain x in (2, 3], range ~[-1.3341e-36, 1.3536e-36]: + * |(lgamma(y+2) - 0.5 * y) / y - s(y)/r(y)| < 2**-120.1 + * with y = x - 2. + */ +static const long double +s0 = -7.72156649015328606065120900824024297e-02L, +s1 = 1.23221687850916448903914170805852253e-01L, +s2 = 5.43673188699937239808255378293820020e-01L, +s3 = 6.31998137119005233383666791176301800e-01L, +s4 = 3.75885340179479850993811501596213763e-01L, +s5 = 1.31572908743275052623410195011261575e-01L, +s6 = 2.82528453299138685507186287149699749e-02L, +s7 = 3.70262021550340817867688714880797019e-03L, +s8 = 2.83374000312371199625774129290973648e-04L, +s9 = 1.15091830239148290758883505582343691e-05L, +s10 = 2.04203474281493971326506384646692446e-07L, +s11 = 9.79544198078992058548607407635645763e-10L, +r1 = 2.58037466655605285937112832039537492e+00L, +r2 = 2.86289413392776399262513849911531180e+00L, +r3 = 1.78691044735267497452847829579514367e+00L, +r4 = 6.89400381446725342846854215600008055e-01L, +r5 = 1.70135865462567955867134197595365343e-01L, +r6 = 2.68794816183964420375498986152766763e-02L, +r7 = 2.64617234244861832870088893332006679e-03L, +r8 = 1.52881761239180800640068128681725702e-04L, +r9 = 4.63264813762296029824851351257638558e-06L, +r10 = 5.89461519146957343083848967333671142e-08L, +r11 = 1.79027678176582527798327441636552968e-10L; +/* + * Domain z in [8, 0x1p70], range ~[-9.8214e-35, 9.8214e-35]: + * |lgamma(x) - (x - 0.5) * (log(x) - 1) - w(1/x)| < 2**-113.0 + */ +static const long double +w0 = 4.18938533204672741780329736405617738e-01L, +w1 = 8.33333333333333333333333333332852026e-02L, +w2 = -2.77777777777777777777777727810123528e-03L, +w3 = 7.93650793650793650791708939493907380e-04L, +w4 = -5.95238095238095234390450004444370959e-04L, +w5 = 8.41750841750837633887817658848845695e-04L, +w6 = -1.91752691752396849943172337347259743e-03L, +w7 = 6.41025640880333069429106541459015557e-03L, +w8 = -2.95506530801732133437990433080327074e-02L, +w9 = 1.79644237328444101596766586979576927e-01L, +w10 = -1.39240539108367641920172649259736394e+00L, +w11 = 1.33987701479007233325288857758641761e+01L, +w12 = -1.56363596431084279780966590116006255e+02L, +w13 = 2.14830978044410267201172332952040777e+03L, +w14 = -3.28636067474227378352761516589092334e+04L, +w15 = 5.06201257747865138432663574251462485e+05L, +w16 = -6.79720123352023636706247599728048344e+06L, +w17 = 6.57556601705472106989497289465949255e+07L, +w18 = -3.26229058141181783534257632389415580e+08L; + +static long double +sin_pil(long double x) +{ + volatile long double vz; + long double y,z; + uint64_t lx, n; + uint16_t hx; + + y = -x; + + vz = y+0x1.p112; + z = vz-0x1.p112; + if (z == y) + return zero; + + vz = y+0x1.p110; + EXTRACT_LDBL128_WORDS(hx,lx,n,vz); + z = vz-0x1.p110; + if (z > y) { + z -= 0.25; + n--; + } + n &= 7; + y = y - z + n * 0.25L; + + switch (n) { + case 0: y = __kernel_sinl(pi*y,zero,0); break; + case 1: + case 2: y = __kernel_cosl(pi*(0.5-y),zero); break; + case 3: + case 4: y = __kernel_sinl(pi*(one-y),zero,0); break; + case 5: + case 6: y = -__kernel_cosl(pi*(y-1.5),zero); break; + default: y = __kernel_sinl(pi*(y-2.0),zero,0); break; + } + return -y; +} + + +long double +lgammal_r(long double x, int *signgamp) +{ + long double nadj,p,p1,p2,p3,q,r,t,w,y,z; + uint64_t llx,lx; + int i; + uint16_t hx; + + EXTRACT_LDBL128_WORDS(hx, lx, llx, x); + + if((hx & 0x7fff) == 0x7fff) { /* erfl(nan)=nan */ + i = (hx>>15)<<1; + return (1-i)+one/x; /* erfl(+-inf)=+-1 */ + } + + /* purge off +-inf, NaN, +-0, tiny and negative arguments */ + *signgamp = 1; + if((hx & 0x7fff) == 0x7fff) /* x is +-Inf or NaN */ + return x*x; + if((hx==0||hx==0x8000)&&lx==0) { + if (hx&0x8000) + *signgamp = -1; + return one/vzero; + } + + /* purge off tiny and negative arguments */ + if(fabsl(x)<0x1p-119L) { + if(hx&0x8000) { + *signgamp = -1; + return -logl(-x); + } else return -logl(x); + } + if(hx&0x8000) { + if(fabsl(x)>=0x1p112) + return one/vzero; + t = sin_pil(x); + if(t==zero) return one/vzero; + nadj = logl(pi/fabsl(t*x)); + if(t=0.7315998077392578) {y = 1-x; i= 0;} + else if(x>=0.2316399812698364) {y= x-(tc-1); i=1;} + else {y = x; i=2;} + } else { + r = 0; + if(x>=1.7316312789916992) {y=2-x;i=0;} + else if(x>=1.2316322326660156) {y=x-tc;i=1;} + else {y=x-1;i=2;} + } + switch(i) { + case 0: + z = y*y; + p1 = a0+z*(a2+z*(a4+z*(a6+z*(a8+z*(a10+z*(a12+z*(a14+z*(a16+ + z*(a18+z*(a20+z*a22)))))))))); + p2 = z*(a1+z*(a3+z*(a5+z*(a7+z*(a9+z*(a11+z*(a13+z*(a15+ + z*(a17+z*(a19+z*(a21+z*a23))))))))))); + p = y*p1+p2; + r += (p-y/2); break; + case 1: + p = t0+y*t1+tt+y*y*(t2+y*(t3+y*(t4+y*(t5+y*(t6+y*(t7+y*(t8+ + y*(t9+y*(t10+y*(t11+y*(t12+y*(t13+y*(t14+y*(t15+y*(t16+ + y*(t17+y*(t18+y*(t19+y*(t20+y*(t21+y*(t22+y*(t23+ + y*(t24+y*(t25+y*(t26+y*(t27+y*(t28+y*(t29+y*(t30+ + y*(t31+y*t32)))))))))))))))))))))))))))))); + r += (tf + p); break; + case 2: + p1 = y*(u0+y*(u1+y*(u2+y*(u3+y*(u4+y*(u5+y*(u6+y*(u7+ + y*(u8+y*(u9+y*u10)))))))))); + p2 = one+y*(v1+y*(v2+y*(v3+y*(v4+y*(v5+y*(v6+y*(v7+ + y*(v8+y*(v9+y*(v10+y*v11)))))))))); + r += (-y/2 + p1/p2); + } + } + else if(x<8) { + i = x; + y = x-i; + p = y*(s0+y*(s1+y*(s2+y*(s3+y*(s4+y*(s5+y*(s6+y*(s7+y*(s8+ + y*(s9+y*(s10+y*s11))))))))))); + q = one+y*(r1+y*(r2+y*(r3+y*(r4+y*(r5+y*(r6+y*(r7+y*(r8+ + y*(r9+y*(r10+y*r11)))))))))); + r = y/2+p/q; + z = 1; /* lgamma(1+s) = log(s) + lgamma(s) */ + switch(i) { + case 7: z *= (y+6); /* FALLTHRU */ + case 6: z *= (y+5); /* FALLTHRU */ + case 5: z *= (y+4); /* FALLTHRU */ + case 4: z *= (y+3); /* FALLTHRU */ + case 3: z *= (y+2); /* FALLTHRU */ + r += logl(z); break; + } + } else if (x < 0x1p119L) { + t = logl(x); + z = one/x; + y = z*z; + w = w0+z*(w1+y*(w2+y*(w3+y*(w4+y*(w5+y*(w6+y*(w7+y*(w8+ + y*(w9+y*(w10+y*(w11+y*(w12+y*(w13+y*(w14+y*(w15+y*(w16+ + y*(w17+y*w18))))))))))))))))); + r = (x-half)*(t-one)+w; + } else + r = x*(logl(x)-1); + if(hx&0x8000) r = nadj - r; + return r; +} diff --git a/libm/upstream-freebsd/lib/msun/src/e_lgamma.c b/libm/upstream-freebsd/lib/msun/src/e_lgamma.c index 4674d9bf0..43f517560 100644 --- a/libm/upstream-freebsd/lib/msun/src/e_lgamma.c +++ b/libm/upstream-freebsd/lib/msun/src/e_lgamma.c @@ -21,6 +21,8 @@ __FBSDID("$FreeBSD$"); * Method: call __ieee754_lgamma_r */ +#include + #include "math.h" #include "math_private.h" @@ -31,3 +33,7 @@ __ieee754_lgamma(double x) { return __ieee754_lgamma_r(x,&signgam); } + +#if (LDBL_MANT_DIG == 53) +__weak_reference(lgamma, lgammal); +#endif diff --git a/libm/upstream-freebsd/lib/msun/src/e_lgamma_r.c b/libm/upstream-freebsd/lib/msun/src/e_lgamma_r.c index 7a95ea47f..a6fd691ff 100644 --- a/libm/upstream-freebsd/lib/msun/src/e_lgamma_r.c +++ b/libm/upstream-freebsd/lib/msun/src/e_lgamma_r.c @@ -83,6 +83,8 @@ __FBSDID("$FreeBSD$"); * */ +#include + #include "math.h" #include "math_private.h" @@ -212,7 +214,11 @@ __ieee754_lgamma_r(double x, int *signgamp) *signgamp = 1; ix = hx&0x7fffffff; if(ix>=0x7ff00000) return x*x; - if((ix|lx)==0) return one/vzero; + if((ix|lx)==0) { + if(hx<0) + *signgamp = -1; + return one/vzero; + } if(ix<0x3b900000) { /* |x|<2**-70, return -log(|x|) */ if(hx<0) { *signgamp = -1; @@ -250,7 +256,7 @@ __ieee754_lgamma_r(double x, int *signgamp) p1 = a0+z*(a2+z*(a4+z*(a6+z*(a8+z*a10)))); p2 = z*(a1+z*(a3+z*(a5+z*(a7+z*(a9+z*a11))))); p = y*p1+p2; - r += (p-0.5*y); break; + r += (p-y/2); break; case 1: z = y*y; w = z*y; @@ -273,11 +279,11 @@ __ieee754_lgamma_r(double x, int *signgamp) r = half*y+p/q; z = one; /* lgamma(1+s) = log(s) + lgamma(s) */ switch(i) { - case 7: z *= (y+6.0); /* FALLTHRU */ - case 6: z *= (y+5.0); /* FALLTHRU */ - case 5: z *= (y+4.0); /* FALLTHRU */ - case 4: z *= (y+3.0); /* FALLTHRU */ - case 3: z *= (y+2.0); /* FALLTHRU */ + case 7: z *= (y+6); /* FALLTHRU */ + case 6: z *= (y+5); /* FALLTHRU */ + case 5: z *= (y+4); /* FALLTHRU */ + case 4: z *= (y+3); /* FALLTHRU */ + case 3: z *= (y+2); /* FALLTHRU */ r += __ieee754_log(z); break; } /* 8.0 <= x < 2**58 */ @@ -293,3 +299,8 @@ __ieee754_lgamma_r(double x, int *signgamp) if(hx<0) r = nadj - r; return r; } + +#if (LDBL_MANT_DIG == 53) +__weak_reference(lgamma_r, lgammal_r); +#endif + diff --git a/libm/upstream-freebsd/lib/msun/src/e_lgammaf_r.c b/libm/upstream-freebsd/lib/msun/src/e_lgammaf_r.c index 9a7ab39dc..9d23053b1 100644 --- a/libm/upstream-freebsd/lib/msun/src/e_lgammaf_r.c +++ b/libm/upstream-freebsd/lib/msun/src/e_lgammaf_r.c @@ -1,5 +1,6 @@ /* e_lgammaf_r.c -- float version of e_lgamma_r.c. * Conversion to float by Ian Lance Taylor, Cygnus Support, ian@cygnus.com. + * Conversion to float fixed By Steven G. Kargl. */ /* @@ -22,72 +23,63 @@ __FBSDID("$FreeBSD$"); static const volatile float vzero = 0; static const float -zero= 0.0000000000e+00, -half= 5.0000000000e-01, /* 0x3f000000 */ -one = 1.0000000000e+00, /* 0x3f800000 */ +zero= 0, +half= 0.5, +one = 1, pi = 3.1415927410e+00, /* 0x40490fdb */ -a0 = 7.7215664089e-02, /* 0x3d9e233f */ -a1 = 3.2246702909e-01, /* 0x3ea51a66 */ -a2 = 6.7352302372e-02, /* 0x3d89f001 */ -a3 = 2.0580807701e-02, /* 0x3ca89915 */ -a4 = 7.3855509982e-03, /* 0x3bf2027e */ -a5 = 2.8905137442e-03, /* 0x3b3d6ec6 */ -a6 = 1.1927076848e-03, /* 0x3a9c54a1 */ -a7 = 5.1006977446e-04, /* 0x3a05b634 */ -a8 = 2.2086278477e-04, /* 0x39679767 */ -a9 = 1.0801156895e-04, /* 0x38e28445 */ -a10 = 2.5214456400e-05, /* 0x37d383a2 */ -a11 = 4.4864096708e-05, /* 0x383c2c75 */ -tc = 1.4616321325e+00, /* 0x3fbb16c3 */ -tf = -1.2148628384e-01, /* 0xbdf8cdcd */ -/* tt = -(tail of tf) */ -tt = 6.6971006518e-09, /* 0x31e61c52 */ -t0 = 4.8383611441e-01, /* 0x3ef7b95e */ -t1 = -1.4758771658e-01, /* 0xbe17213c */ -t2 = 6.4624942839e-02, /* 0x3d845a15 */ -t3 = -3.2788541168e-02, /* 0xbd064d47 */ -t4 = 1.7970675603e-02, /* 0x3c93373d */ -t5 = -1.0314224288e-02, /* 0xbc28fcfe */ -t6 = 6.1005386524e-03, /* 0x3bc7e707 */ -t7 = -3.6845202558e-03, /* 0xbb7177fe */ -t8 = 2.2596477065e-03, /* 0x3b141699 */ -t9 = -1.4034647029e-03, /* 0xbab7f476 */ -t10 = 8.8108185446e-04, /* 0x3a66f867 */ -t11 = -5.3859531181e-04, /* 0xba0d3085 */ -t12 = 3.1563205994e-04, /* 0x39a57b6b */ -t13 = -3.1275415677e-04, /* 0xb9a3f927 */ -t14 = 3.3552918467e-04, /* 0x39afe9f7 */ -u0 = -7.7215664089e-02, /* 0xbd9e233f */ -u1 = 6.3282704353e-01, /* 0x3f2200f4 */ -u2 = 1.4549225569e+00, /* 0x3fba3ae7 */ -u3 = 9.7771751881e-01, /* 0x3f7a4bb2 */ -u4 = 2.2896373272e-01, /* 0x3e6a7578 */ -u5 = 1.3381091878e-02, /* 0x3c5b3c5e */ -v1 = 2.4559779167e+00, /* 0x401d2ebe */ -v2 = 2.1284897327e+00, /* 0x4008392d */ -v3 = 7.6928514242e-01, /* 0x3f44efdf */ -v4 = 1.0422264785e-01, /* 0x3dd572af */ -v5 = 3.2170924824e-03, /* 0x3b52d5db */ -s0 = -7.7215664089e-02, /* 0xbd9e233f */ -s1 = 2.1498242021e-01, /* 0x3e5c245a */ -s2 = 3.2577878237e-01, /* 0x3ea6cc7a */ -s3 = 1.4635047317e-01, /* 0x3e15dce6 */ -s4 = 2.6642270386e-02, /* 0x3cda40e4 */ -s5 = 1.8402845599e-03, /* 0x3af135b4 */ -s6 = 3.1947532989e-05, /* 0x3805ff67 */ -r1 = 1.3920053244e+00, /* 0x3fb22d3b */ -r2 = 7.2193557024e-01, /* 0x3f38d0c5 */ -r3 = 1.7193385959e-01, /* 0x3e300f6e */ -r4 = 1.8645919859e-02, /* 0x3c98bf54 */ -r5 = 7.7794247773e-04, /* 0x3a4beed6 */ -r6 = 7.3266842264e-06, /* 0x36f5d7bd */ -w0 = 4.1893854737e-01, /* 0x3ed67f1d */ -w1 = 8.3333335817e-02, /* 0x3daaaaab */ -w2 = -2.7777778450e-03, /* 0xbb360b61 */ -w3 = 7.9365057172e-04, /* 0x3a500cfd */ -w4 = -5.9518753551e-04, /* 0xba1c065c */ -w5 = 8.3633989561e-04, /* 0x3a5b3dd2 */ -w6 = -1.6309292987e-03; /* 0xbad5c4e8 */ +/* + * Domain y in [0x1p-27, 0.27], range ~[-3.4599e-10, 3.4590e-10]: + * |(lgamma(2 - y) + 0.5 * y) / y - a(y)| < 2**-31.4 + */ +a0 = 7.72156641e-02, /* 0x3d9e233f */ +a1 = 3.22467119e-01, /* 0x3ea51a69 */ +a2 = 6.73484802e-02, /* 0x3d89ee00 */ +a3 = 2.06395667e-02, /* 0x3ca9144f */ +a4 = 6.98275631e-03, /* 0x3be4cf9b */ +a5 = 4.11768444e-03, /* 0x3b86eda4 */ +/* + * Domain x in [tc-0.24, tc+0.28], range ~[-5.6577e-10, 5.5677e-10]: + * |(lgamma(x) - tf) - t(x - tc)| < 2**-30.8. + */ +tc = 1.46163213e+00, /* 0x3fbb16c3 */ +tf = -1.21486291e-01, /* 0xbdf8cdce */ +t0 = -2.94064460e-11, /* 0xae0154b7 */ +t1 = -2.35939837e-08, /* 0xb2caabb8 */ +t2 = 4.83836412e-01, /* 0x3ef7b968 */ +t3 = -1.47586212e-01, /* 0xbe1720d7 */ +t4 = 6.46013096e-02, /* 0x3d844db1 */ +t5 = -3.28450352e-02, /* 0xbd068884 */ +t6 = 1.86483748e-02, /* 0x3c98c47a */ +t7 = -9.89206228e-03, /* 0xbc221251 */ +/* + * Domain y in [-0.1, 0.232], range ~[-8.4931e-10, 8.7794e-10]: + * |(lgamma(1 + y) + 0.5 * y) / y - u(y) / v(y)| < 2**-31.2 + */ +u0 = -7.72156641e-02, /* 0xbd9e233f */ +u1 = 7.36789703e-01, /* 0x3f3c9e40 */ +u2 = 4.95649040e-01, /* 0x3efdc5b6 */ +v1 = 1.10958421e+00, /* 0x3f8e06db */ +v2 = 2.10598111e-01, /* 0x3e57a708 */ +v3 = -1.02995494e-02, /* 0xbc28bf71 */ +/* + * Domain x in (2, 3], range ~[-5.5189e-11, 5.2317e-11]: + * |(lgamma(y+2) - 0.5 * y) / y - s(y)/r(y)| < 2**-35.0 + * with y = x - 2. + */ +s0 = -7.72156641e-02, /* 0xbd9e233f */ +s1 = 2.69987404e-01, /* 0x3e8a3bca */ +s2 = 1.42851010e-01, /* 0x3e124789 */ +s3 = 1.19389519e-02, /* 0x3c439b98 */ +r1 = 6.79650068e-01, /* 0x3f2dfd8c */ +r2 = 1.16058730e-01, /* 0x3dedb033 */ +r3 = 3.75673687e-03, /* 0x3b763396 */ +/* + * Domain z in [8, 0x1p24], range ~[-1.2640e-09, 1.2640e-09]: + * |lgamma(x) - (x - 0.5) * (log(x) - 1) - w(1/x)| < 2**-29.6. + */ +w0 = 4.18938547e-01, /* 0x3ed67f1d */ +w1 = 8.33332464e-02, /* 0x3daaaa9f */ +w2 = -2.76129087e-03; /* 0xbb34f6c6 */ static float sin_pif(float x) @@ -140,7 +132,11 @@ __ieee754_lgammaf_r(float x, int *signgamp) *signgamp = 1; ix = hx&0x7fffffff; if(ix>=0x7f800000) return x*x; - if(ix==0) return one/vzero; + if(ix==0) { + if(hx<0) + *signgamp = -1; + return one/vzero; + } if(ix<0x35000000) { /* |x|<2**-21, return -log(|x|) */ if(hx<0) { *signgamp = -1; @@ -168,55 +164,50 @@ __ieee754_lgammaf_r(float x, int *signgamp) else {y = x; i=2;} } else { r = zero; - if(ix>=0x3fdda618) {y=(float)2.0-x;i=0;} /* [1.7316,2] */ + if(ix>=0x3fdda618) {y=2-x;i=0;} /* [1.7316,2] */ else if(ix>=0x3F9da620) {y=x-tc;i=1;} /* [1.23,1.73] */ else {y=x-one;i=2;} } switch(i) { case 0: z = y*y; - p1 = a0+z*(a2+z*(a4+z*(a6+z*(a8+z*a10)))); - p2 = z*(a1+z*(a3+z*(a5+z*(a7+z*(a9+z*a11))))); + p1 = a0+z*(a2+z*a4); + p2 = z*(a1+z*(a3+z*a5)); p = y*p1+p2; - r += (p-(float)0.5*y); break; + r += (p-y/2); break; case 1: - z = y*y; - w = z*y; - p1 = t0+w*(t3+w*(t6+w*(t9 +w*t12))); /* parallel comp */ - p2 = t1+w*(t4+w*(t7+w*(t10+w*t13))); - p3 = t2+w*(t5+w*(t8+w*(t11+w*t14))); - p = z*p1-(tt-w*(p2+y*p3)); + p = t0+y*t1+y*y*(t2+y*(t3+y*(t4+y*(t5+y*(t6+y*t7))))); r += (tf + p); break; case 2: - p1 = y*(u0+y*(u1+y*(u2+y*(u3+y*(u4+y*u5))))); - p2 = one+y*(v1+y*(v2+y*(v3+y*(v4+y*v5)))); - r += (-(float)0.5*y + p1/p2); + p1 = y*(u0+y*(u1+y*u2)); + p2 = one+y*(v1+y*(v2+y*v3)); + r += (p1/p2-y/2); } } else if(ix<0x41000000) { /* x < 8.0 */ - i = (int)x; - y = x-(float)i; - p = y*(s0+y*(s1+y*(s2+y*(s3+y*(s4+y*(s5+y*s6)))))); - q = one+y*(r1+y*(r2+y*(r3+y*(r4+y*(r5+y*r6))))); - r = half*y+p/q; + i = x; + y = x-i; + p = y*(s0+y*(s1+y*(s2+y*s3))); + q = one+y*(r1+y*(r2+y*r3)); + r = y/2+p/q; z = one; /* lgamma(1+s) = log(s) + lgamma(s) */ switch(i) { - case 7: z *= (y+(float)6.0); /* FALLTHRU */ - case 6: z *= (y+(float)5.0); /* FALLTHRU */ - case 5: z *= (y+(float)4.0); /* FALLTHRU */ - case 4: z *= (y+(float)3.0); /* FALLTHRU */ - case 3: z *= (y+(float)2.0); /* FALLTHRU */ + case 7: z *= (y+6); /* FALLTHRU */ + case 6: z *= (y+5); /* FALLTHRU */ + case 5: z *= (y+4); /* FALLTHRU */ + case 4: z *= (y+3); /* FALLTHRU */ + case 3: z *= (y+2); /* FALLTHRU */ r += __ieee754_logf(z); break; } - /* 8.0 <= x < 2**58 */ - } else if (ix < 0x5c800000) { + /* 8.0 <= x < 2**24 */ + } else if (ix < 0x4b800000) { t = __ieee754_logf(x); z = one/x; y = z*z; - w = w0+z*(w1+y*(w2+y*(w3+y*(w4+y*(w5+y*w6))))); + w = w0+z*(w1+y*w2); r = (x-half)*(t-one)+w; } else - /* 2**58 <= x <= inf */ + /* 2**24 <= x <= inf */ r = x*(__ieee754_logf(x)-one); if(hx<0) r = nadj - r; return r; diff --git a/libm/upstream-freebsd/lib/msun/src/e_lgammal.c b/libm/upstream-freebsd/lib/msun/src/e_lgammal.c new file mode 100644 index 000000000..ebc2fc78c --- /dev/null +++ b/libm/upstream-freebsd/lib/msun/src/e_lgammal.c @@ -0,0 +1,25 @@ +/* @(#)e_lgamma.c 1.3 95/01/18 */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunSoft, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "math.h" +#include "math_private.h" + +extern int signgam; + +long double +lgammal(long double x) +{ + return lgammal_r(x,&signgam); +} diff --git a/libm/upstream-freebsd/lib/msun/src/imprecise.c b/libm/upstream-freebsd/lib/msun/src/imprecise.c index 92fb2d06e..08cd23980 100644 --- a/libm/upstream-freebsd/lib/msun/src/imprecise.c +++ b/libm/upstream-freebsd/lib/msun/src/imprecise.c @@ -60,5 +60,4 @@ DECLARE_WEAK(powl); long double imprecise_ ## f ## l(long double v) { return f(v); }\ DECLARE_WEAK(f ## l) -DECLARE_IMPRECISE(lgamma); DECLARE_IMPRECISE(tgamma); diff --git a/tests/math_test.cpp b/tests/math_test.cpp index 2203db9f4..d0fbc1f8f 100644 --- a/tests/math_test.cpp +++ b/tests/math_test.cpp @@ -1188,12 +1188,51 @@ TEST(math, lgamma_r) { ASSERT_EQ(1, sign); } +TEST(math, lgamma_r_17471883) { + int sign; + + sign = 0; + ASSERT_DOUBLE_EQ(HUGE_VAL, lgamma_r(0.0, &sign)); + ASSERT_EQ(1, sign); + sign = 0; + ASSERT_DOUBLE_EQ(HUGE_VAL, lgamma_r(-0.0, &sign)); + ASSERT_EQ(-1, sign); +} + TEST(math, lgammaf_r) { int sign; ASSERT_FLOAT_EQ(logf(24.0f), lgammaf_r(5.0f, &sign)); ASSERT_EQ(1, sign); } +TEST(math, lgammaf_r_17471883) { + int sign; + + sign = 0; + ASSERT_FLOAT_EQ(HUGE_VALF, lgammaf_r(0.0f, &sign)); + ASSERT_EQ(1, sign); + sign = 0; + ASSERT_FLOAT_EQ(HUGE_VALF, lgammaf_r(-0.0f, &sign)); + ASSERT_EQ(-1, sign); +} + +TEST(math, lgammal_r) { + int sign; + ASSERT_DOUBLE_EQ(log(24.0L), lgamma_r(5.0L, &sign)); + ASSERT_EQ(1, sign); +} + +TEST(math, lgammal_r_17471883) { + int sign; + + sign = 0; + ASSERT_DOUBLE_EQ(HUGE_VAL, lgammal_r(0.0L, &sign)); + ASSERT_EQ(1, sign); + sign = 0; + ASSERT_DOUBLE_EQ(HUGE_VAL, lgammal_r(-0.0L, &sign)); + ASSERT_EQ(-1, sign); +} + TEST(math, tgamma) { ASSERT_DOUBLE_EQ(24.0, tgamma(5.0)); } From cbb6fcca09fd5593e36b2aa7643602f31700d1d1 Mon Sep 17 00:00:00 2001 From: Jake Weinstein Date: Sun, 16 Aug 2015 00:44:40 +0000 Subject: [PATCH 81/89] libc: remove bcopy from memmove on 64-bit architectures * bcopy is deprecated on LP64 by the following commit: https://android.googlesource.com/platform/bionic/+/ce9ce28e5d760e32ab6c894dfaf7b8dad6de7ff6 Change-Id: I6849916f0ec4a2d0db9a360999ad1dc8edda952b --- libc/arch-arm64/generic/bionic/memmove.S | 16 ++-------------- libc/arch-x86_64/string/sse2-memmove-slm.S | 3 --- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/libc/arch-arm64/generic/bionic/memmove.S b/libc/arch-arm64/generic/bionic/memmove.S index 8b366a314..739ce4982 100644 --- a/libc/arch-arm64/generic/bionic/memmove.S +++ b/libc/arch-arm64/generic/bionic/memmove.S @@ -35,10 +35,6 @@ #include /* Parameters and result. */ -#ifdef BCOPY -#define origdstin x1 -#define origsrc x0 -#endif #define dstin x0 #define src x1 #define count x2 @@ -59,13 +55,7 @@ #define D_l x13 #define D_h x14 -#ifdef BCOPY -ENTRY(bcopy) - /* Swap src and dst so that a branch to memcpy doesn't cause issues. */ - mov tmp1, origsrc - mov origsrc, origdstin - mov origdstin, tmp1 -#elif defined(WMEMMOVE) +#if defined(WMEMMOVE) ENTRY(wmemmove) lsl count, count, #2 #else @@ -332,9 +322,7 @@ ENTRY(memmove) tst count, #0x3f b.ne .Ltail63down ret -#ifdef BCOPY -END(bcopy) -#elif defined(WMEMMOVE) +#if defined(WMEMMOVE) END(wmemmove) #else END(memmove) diff --git a/libc/arch-x86_64/string/sse2-memmove-slm.S b/libc/arch-x86_64/string/sse2-memmove-slm.S index 0dbffad2c..6a5afd610 100644 --- a/libc/arch-x86_64/string/sse2-memmove-slm.S +++ b/libc/arch-x86_64/string/sse2-memmove-slm.S @@ -91,9 +91,6 @@ name: \ .section .text.sse2,"ax",@progbits ENTRY (MEMMOVE) ENTRANCE -#ifdef USE_AS_BCOPY - xchg %rsi, %rdi -#endif mov %rdi, %rax /* Check whether we should copy backward or forward. */ From 0196aa9b4cb63857d702c7c49ba0e3e422ef1a51 Mon Sep 17 00:00:00 2001 From: Steve Kondik Date: Wed, 19 Aug 2015 15:48:23 -0700 Subject: [PATCH 82/89] Revert "Add optimized cortex-a7/cortex-a53 memset/memcpy." * Replacing with updated version from AOSP. This reverts commit a2cd3dd0ffbc61f6f578ec1777090db5374c28ea. Change-Id: I8feefd402d85395d489d067d002806658ac6d734 --- .../arch-arm/cortex-a53/bionic/__strcat_chk.S | 214 ------------------ .../arch-arm/cortex-a53/bionic/__strcpy_chk.S | 179 --------------- libc/arch-arm/cortex-a53/bionic/memcpy.S | 102 --------- libc/arch-arm/cortex-a53/bionic/memcpy_base.S | 148 ------------ libc/arch-arm/cortex-a53/cortex-a53.mk | 16 +- libc/arch-arm/cortex-a7/bionic/memset.S | 180 --------------- 6 files changed, 5 insertions(+), 834 deletions(-) delete mode 100644 libc/arch-arm/cortex-a53/bionic/__strcat_chk.S delete mode 100644 libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S delete mode 100644 libc/arch-arm/cortex-a53/bionic/memcpy.S delete mode 100644 libc/arch-arm/cortex-a53/bionic/memcpy_base.S delete mode 100644 libc/arch-arm/cortex-a7/bionic/memset.S diff --git a/libc/arch-arm/cortex-a53/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a53/bionic/__strcat_chk.S deleted file mode 100644 index 476e2aece..000000000 --- a/libc/arch-arm/cortex-a53/bionic/__strcat_chk.S +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -#include - - .syntax unified - - .thumb - .thumb_func - -// Get the length of src string, then get the source of the dst string. -// Check that the two lengths together don't exceed the threshold, then -// do a memcpy of the data. -ENTRY(__strcat_chk) - pld [r0, #0] - push {r0, lr} - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - push {r4, r5} - .cfi_adjust_cfa_offset 8 - .cfi_rel_offset r4, 0 - .cfi_rel_offset r5, 4 - - mov lr, r2 - - // Save the dst register to r5 - mov r5, r0 - - // Zero out r4 - eor r4, r4, r4 - - // r1 contains the address of the string to count. -.L_strlen_start: - mov r0, r1 - ands r3, r1, #7 - beq .L_mainloop - - // Align to a double word (64 bits). - rsb r3, r3, #8 - lsls ip, r3, #31 - beq .L_align_to_32 - - ldrb r2, [r1], #1 - cbz r2, .L_update_count_and_finish - -.L_align_to_32: - bcc .L_align_to_64 - ands ip, r3, #2 - beq .L_align_to_64 - - ldrb r2, [r1], #1 - cbz r2, .L_update_count_and_finish - ldrb r2, [r1], #1 - cbz r2, .L_update_count_and_finish - -.L_align_to_64: - tst r3, #4 - beq .L_mainloop - ldr r3, [r1], #4 - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne .L_zero_in_second_register - - .p2align 2 -.L_mainloop: - ldrd r2, r3, [r1], #8 - - pld [r1, #64] - - sub ip, r2, #0x01010101 - bic ip, ip, r2 - ands ip, ip, #0x80808080 - bne .L_zero_in_first_register - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne .L_zero_in_second_register - b .L_mainloop - -.L_update_count_and_finish: - sub r3, r1, r0 - sub r3, r3, #1 - b .L_finish - -.L_zero_in_first_register: - sub r3, r1, r0 - lsls r2, ip, #17 - bne .L_sub8_and_finish - bcs .L_sub7_and_finish - lsls ip, ip, #1 - bne .L_sub6_and_finish - - sub r3, r3, #5 - b .L_finish - -.L_sub8_and_finish: - sub r3, r3, #8 - b .L_finish - -.L_sub7_and_finish: - sub r3, r3, #7 - b .L_finish - -.L_sub6_and_finish: - sub r3, r3, #6 - b .L_finish - -.L_zero_in_second_register: - sub r3, r1, r0 - lsls r2, ip, #17 - bne .L_sub4_and_finish - bcs .L_sub3_and_finish - lsls ip, ip, #1 - bne .L_sub2_and_finish - - sub r3, r3, #1 - b .L_finish - -.L_sub4_and_finish: - sub r3, r3, #4 - b .L_finish - -.L_sub3_and_finish: - sub r3, r3, #3 - b .L_finish - -.L_sub2_and_finish: - sub r3, r3, #2 - -.L_finish: - cmp r4, #0 - bne .L_strlen_done - - // Time to get the dst string length. - mov r1, r5 - - // Save the original source address to r5. - mov r5, r0 - - // Save the current length (adding 1 for the terminator). - add r4, r3, #1 - b .L_strlen_start - - // r0 holds the pointer to the dst string. - // r3 holds the dst string length. - // r4 holds the src string length + 1. -.L_strlen_done: - add r2, r3, r4 - cmp r2, lr - bhi __strcat_chk_failed - - // Set up the registers for the memcpy code. - mov r1, r5 - pld [r1, #64] - mov r2, r4 - add r0, r0, r3 - pop {r4, r5} -END(__strcat_chk) - -#define MEMCPY_BASE __strcat_chk_memcpy_base -#include "memcpy_base.S" - -ENTRY_PRIVATE(__strcat_chk_failed) - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - .cfi_adjust_cfa_offset 8 - .cfi_rel_offset r4, 0 - .cfi_rel_offset r5, 4 - - ldr r0, error_message - ldr r1, error_code -1: - add r0, pc - bl __fortify_chk_fail -error_code: - .word BIONIC_EVENT_STRCAT_BUFFER_OVERFLOW -error_message: - .word error_string-(1b+4) -END(__strcat_chk_failed) - - .data -error_string: - .string "strcat: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S deleted file mode 100644 index 6078a51b8..000000000 --- a/libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -#include - - .syntax unified - - .thumb - .thumb_func - -// Get the length of the source string first, then do a memcpy of the data -// instead of a strcpy. -ENTRY(__strcpy_chk) - pld [r0, #0] - push {r0, lr} - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - - mov lr, r2 - mov r0, r1 - - ands r3, r1, #7 - beq .L_mainloop - - // Align to a double word (64 bits). - rsb r3, r3, #8 - lsls ip, r3, #31 - beq .L_align_to_32 - - ldrb r2, [r0], #1 - cbz r2, .L_update_count_and_finish - -.L_align_to_32: - bcc .L_align_to_64 - ands ip, r3, #2 - beq .L_align_to_64 - - ldrb r2, [r0], #1 - cbz r2, .L_update_count_and_finish - ldrb r2, [r0], #1 - cbz r2, .L_update_count_and_finish - -.L_align_to_64: - tst r3, #4 - beq .L_mainloop - ldr r3, [r0], #4 - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne .L_zero_in_second_register - - .p2align 2 -.L_mainloop: - ldrd r2, r3, [r0], #8 - - pld [r0, #64] - - sub ip, r2, #0x01010101 - bic ip, ip, r2 - ands ip, ip, #0x80808080 - bne .L_zero_in_first_register - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne .L_zero_in_second_register - b .L_mainloop - -.L_update_count_and_finish: - sub r3, r0, r1 - sub r3, r3, #1 - b .L_check_size - -.L_zero_in_first_register: - sub r3, r0, r1 - lsls r2, ip, #17 - bne .L_sub8_and_finish - bcs .L_sub7_and_finish - lsls ip, ip, #1 - bne .L_sub6_and_finish - - sub r3, r3, #5 - b .L_check_size - -.L_sub8_and_finish: - sub r3, r3, #8 - b .L_check_size - -.L_sub7_and_finish: - sub r3, r3, #7 - b .L_check_size - -.L_sub6_and_finish: - sub r3, r3, #6 - b .L_check_size - -.L_zero_in_second_register: - sub r3, r0, r1 - lsls r2, ip, #17 - bne .L_sub4_and_finish - bcs .L_sub3_and_finish - lsls ip, ip, #1 - bne .L_sub2_and_finish - - sub r3, r3, #1 - b .L_check_size - -.L_sub4_and_finish: - sub r3, r3, #4 - b .L_check_size - -.L_sub3_and_finish: - sub r3, r3, #3 - b .L_check_size - -.L_sub2_and_finish: - sub r3, r3, #2 - -.L_check_size: - pld [r1, #0] - pld [r1, #64] - ldr r0, [sp] - cmp r3, lr - bhs __strcpy_chk_failed - - // Add 1 for copy length to get the string terminator. - add r2, r3, #1 -END(__strcpy_chk) - -#define MEMCPY_BASE __strcpy_chk_memcpy_base -#include "memcpy_base.S" - -ENTRY_PRIVATE(__strcpy_chk_failed) - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - - ldr r0, error_message - ldr r1, error_code -1: - add r0, pc - bl __fortify_chk_fail -error_code: - .word BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW -error_message: - .word error_string-(1b+4) -END(__strcpy_chk_failed) - - .data -error_string: - .string "strcpy: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a53/bionic/memcpy.S b/libc/arch-arm/cortex-a53/bionic/memcpy.S deleted file mode 100644 index 1b29543e7..000000000 --- a/libc/arch-arm/cortex-a53/bionic/memcpy.S +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (C) 2008 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -/* - * Copyright (c) 2013 ARM Ltd - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the company may not be used to endorse or promote - * products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -// Prototype: void *memcpy (void *dst, const void *src, size_t count). - -#include -#include - - .text - .syntax unified - .fpu neon - -ENTRY(__memcpy_chk) - cmp r2, r3 - bhi __memcpy_chk_fail - - // Fall through to memcpy... -END(__memcpy_chk) - -ENTRY(memcpy) - pld [r1, #64] - push {r0, lr} - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 -END(memcpy) - -#define MEMCPY_BASE __memcpy_base -#include "memcpy_base.S" - -ENTRY_PRIVATE(__memcpy_chk_fail) - // Preserve lr for backtrace. - push {lr} - .cfi_def_cfa_offset 4 - .cfi_rel_offset lr, 0 - - ldr r0, error_message - ldr r1, error_code -1: - add r0, pc - bl __fortify_chk_fail -error_code: - .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW -error_message: - .word error_string-(1b+8) -END(__memcpy_chk_fail) - - .data -error_string: - .string "memcpy: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a53/bionic/memcpy_base.S b/libc/arch-arm/cortex-a53/bionic/memcpy_base.S deleted file mode 100644 index b54f141cc..000000000 --- a/libc/arch-arm/cortex-a53/bionic/memcpy_base.S +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (C) 2008 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -/* - * Copyright (c) 2013 ARM Ltd - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the company may not be used to endorse or promote - * products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -ENTRY_PRIVATE(MEMCPY_BASE) - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - - // Assumes that n >= 0, and dst, src are valid pointers. - cmp r2, #16 - blo .L_copy_less_than_16_unknown_align - -.L_copy_unknown_alignment: - // Unknown alignment of src and dst. - // Assumes that the first few bytes have already been prefetched. - - // Align destination to 128 bits. The mainloop store instructions - // require this alignment or they will throw an exception. - rsb r3, r0, #0 - ands r3, r3, #0xF - beq 2f - - // Copy up to 15 bytes (count in r3). - sub r2, r2, r3 - movs ip, r3, lsl #31 - - itt mi - ldrbmi lr, [r1], #1 - strbmi lr, [r0], #1 - itttt cs - ldrbcs ip, [r1], #1 - ldrbcs lr, [r1], #1 - strbcs ip, [r0], #1 - strbcs lr, [r0], #1 - - movs ip, r3, lsl #29 - bge 1f - // Copies 4 bytes, dst 32 bits aligned before, at least 64 bits after. - vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! - vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! -1: bcc 2f - // Copies 8 bytes, dst 64 bits aligned before, at least 128 bits after. - vld1.8 {d0}, [r1]! - vst1.8 {d0}, [r0, :64]! - -2: // Make sure we have at least 64 bytes to copy. - subs r2, r2, #64 - blo 2f - -1: // The main loop copies 64 bytes at a time. - vld1.8 {d0 - d3}, [r1]! - vld1.8 {d4 - d7}, [r1]! - subs r2, r2, #64 - vstmia r0!, {d0 - d7} - pld [r1, #(64*10)] - bhs 1b - -2: // Fix-up the remaining count and make sure we have >= 32 bytes left. - adds r2, r2, #32 - blo 3f - - // 32 bytes. These cache lines were already preloaded. - vld1.8 {d0 - d3}, [r1]! - sub r2, r2, #32 - vst1.8 {d0 - d3}, [r0, :128]! -3: // Less than 32 left. - add r2, r2, #32 - tst r2, #0x10 - beq .L_copy_less_than_16_unknown_align - // Copies 16 bytes, destination 128 bits aligned. - vld1.8 {d0, d1}, [r1]! - vst1.8 {d0, d1}, [r0, :128]! - -.L_copy_less_than_16_unknown_align: - // Copy up to 15 bytes (count in r2). - movs ip, r2, lsl #29 - bcc 1f - vld1.8 {d0}, [r1]! - vst1.8 {d0}, [r0]! -1: bge 2f - vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! - vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! - -2: // Copy 0 to 4 bytes. - lsls r2, r2, #31 - itt ne - ldrbne lr, [r1], #1 - strbne lr, [r0], #1 - itttt cs - ldrbcs ip, [r1], #1 - ldrbcs lr, [r1] - strbcs ip, [r0], #1 - strbcs lr, [r0] - - pop {r0, pc} -END(MEMCPY_BASE) diff --git a/libc/arch-arm/cortex-a53/cortex-a53.mk b/libc/arch-arm/cortex-a53/cortex-a53.mk index 3f75ebbfc..e6fc66d6e 100644 --- a/libc/arch-arm/cortex-a53/cortex-a53.mk +++ b/libc/arch-arm/cortex-a53/cortex-a53.mk @@ -1,18 +1,12 @@ libc_bionic_src_files_arm += \ - arch-arm/cortex-a53/bionic/memcpy.S \ - arch-arm/cortex-a53/bionic/__strcat_chk.S \ - arch-arm/cortex-a53/bionic/__strcpy_chk.S \ - -libc_bionic_src_files_arm += \ - arch-arm/cortex-a7/bionic/memset.S \ - -libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memchr.S \ + arch-arm/cortex-a15/bionic/memcpy.S \ + arch-arm/cortex-a15/bionic/memset.S \ arch-arm/cortex-a15/bionic/strcat.S \ arch-arm/cortex-a15/bionic/strcmp.S \ arch-arm/cortex-a15/bionic/strcpy.S \ arch-arm/cortex-a15/bionic/strlen.S \ arch-arm/cortex-a15/bionic/stpcpy.S \ + arch-arm/cortex-a15/bionic/__strcat_chk.S \ + arch-arm/cortex-a15/bionic/__strcpy_chk.S \ arch-arm/krait/bionic/memmove.S - -libc_bionic_src_files_arm += \ - arch-arm/cortex-a15/bionic/memchr.S diff --git a/libc/arch-arm/cortex-a7/bionic/memset.S b/libc/arch-arm/cortex-a7/bionic/memset.S deleted file mode 100644 index 6365b066a..000000000 --- a/libc/arch-arm/cortex-a7/bionic/memset.S +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -#include -#include - - /* - * Optimized memset() for ARM. - * - * memset() returns its first argument. - */ - - .fpu neon - .syntax unified - -ENTRY(__memset_chk) - cmp r2, r3 - bls .L_done - - // Preserve lr for backtrace. - push {lr} - .cfi_def_cfa_offset 4 - .cfi_rel_offset lr, 0 - - ldr r0, error_message - ldr r1, error_code -1: - add r0, pc - bl __fortify_chk_fail -error_code: - .word BIONIC_EVENT_MEMSET_BUFFER_OVERFLOW -error_message: - .word error_string-(1b+8) -END(__memset_chk) - -ENTRY(bzero) - mov r2, r1 - mov r1, #0 -.L_done: - // Fall through to memset... -END(bzero) - -ENTRY(memset) - mov r3, r0 - // At this point only d0, d1 are going to be used below. - vdup.8 q0, r1 - cmp r2, #16 - blo .L_set_less_than_16_unknown_align - -.L_check_alignment: - // Align destination to a double word to avoid the store crossing - // a cache line boundary. - ands ip, r3, #7 - bne .L_do_double_word_align - -.L_double_word_aligned: - // Duplicate since the less than 64 can use d2, d3. - vmov q1, q0 - subs r2, #64 - blo .L_set_less_than_64 - - // Duplicate the copy value so that we can store 64 bytes at a time. - vmov q2, q0 - vmov q3, q0 - -1: // Main loop stores 64 bytes at a time. - subs r2, #64 - vstmia r3!, {d0 - d7} - bge 1b - -.L_set_less_than_64: - // Restore r2 to the count of bytes left to set. - add r2, #64 - lsls ip, r2, #27 - bcc .L_set_less_than_32 - // Set 32 bytes. - vstmia r3!, {d0 - d3} - -.L_set_less_than_32: - bpl .L_set_less_than_16 - // Set 16 bytes. - vstmia r3!, {d0, d1} - -.L_set_less_than_16: - // Less than 16 bytes to set. - lsls ip, r2, #29 - bcc .L_set_less_than_8 - - // Set 8 bytes. - vstmia r3!, {d0} - -.L_set_less_than_8: - bpl .L_set_less_than_4 - // Set 4 bytes - vst1.32 {d0[0]}, [r3]! - -.L_set_less_than_4: - lsls ip, r2, #31 - it ne - strbne r1, [r3], #1 - itt cs - strbcs r1, [r3], #1 - strbcs r1, [r3] - bx lr - -.L_do_double_word_align: - rsb ip, ip, #8 - sub r2, r2, ip - - // Do this comparison now, otherwise we'll need to save a - // register to the stack since we've used all available - // registers. - cmp ip, #4 - blo 1f - - // Need to do a four byte copy. - movs ip, ip, lsl #31 - it mi - strbmi r1, [r3], #1 - itt cs - strbcs r1, [r3], #1 - strbcs r1, [r3], #1 - vst1.32 {d0[0]}, [r3]! - b .L_double_word_aligned - -1: - // No four byte copy. - movs ip, ip, lsl #31 - it mi - strbmi r1, [r3], #1 - itt cs - strbcs r1, [r3], #1 - strbcs r1, [r3], #1 - b .L_double_word_aligned - -.L_set_less_than_16_unknown_align: - // Set up to 15 bytes. - movs ip, r2, lsl #29 - bcc 1f - vst1.8 {d0}, [r3]! -1: bge 2f - vst1.32 {d0[0]}, [r3]! -2: movs ip, r2, lsl #31 - it mi - strbmi r1, [r3], #1 - itt cs - strbcs r1, [r3], #1 - strbcs r1, [r3], #1 - bx lr -END(memset) - - .data -error_string: - .string "memset: prevented write past end of buffer" From 49bb8417bcba13730c26ddee4e1c65f1f08556f5 Mon Sep 17 00:00:00 2001 From: Steve Kondik Date: Wed, 19 Aug 2015 15:54:05 -0700 Subject: [PATCH 83/89] Revert "Replace memcpy with the cortex-strings implementation" * Prepare for updates from AOSP. This reverts commit 554721b671b5e9216aaa0d7603fadaca1c592b3e. Change-Id: Id39a881bb114d154b32ee8d98fe5b29384b4466f --- libc/arch-arm/cortex-a15/bionic/memcpy.S | 669 +++-------------------- 1 file changed, 63 insertions(+), 606 deletions(-) diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy.S b/libc/arch-arm/cortex-a15/bionic/memcpy.S index 5591d78cd..410b663e2 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy.S @@ -1,623 +1,80 @@ -/* Copyright (c) 2013, Linaro Limited - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of Linaro Limited nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. */ - /* - This memcpy routine is optimised for Cortex-A15 cores and takes advantage - of VFP or NEON when built with the appropriate flags. - - Assumptions: - - ARMv6 (ARMv7-a if using Neon) - ARM state - Unaligned accesses - - Ported to work inside Bionic by Bernhard Rosenkränzer + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +// Prototype: void *memcpy (void *dst, const void *src, size_t count). + #include #include - .syntax unified - /* This implementation requires ARM state. */ - .arm - -#ifdef __ARM_NEON__ - - .fpu neon - .arch armv7-a -# define FRAME_SIZE 4 -# define USE_VFP -# define USE_NEON - -#elif !defined (__SOFTFP__) -#error VFP - - .arch armv6 - .fpu vfpv2 -# define FRAME_SIZE 32 -# define USE_VFP - -#else -#error NO FPU - .arch armv6 -# define FRAME_SIZE 32 - -#endif - -/* Old versions of GAS incorrectly implement the NEON align semantics. */ -#ifdef BROKEN_ASM_NEON_ALIGN -#define ALIGN(addr, align) addr,:align -#else -#define ALIGN(addr, align) addr:align -#endif - -#define PC_OFFSET 8 /* PC pipeline compensation. */ -#define INSN_SIZE 4 - -/* Call parameters. */ -#define dstin r0 -#define src r1 -#define count r2 - -/* Locals. */ -#define tmp1 r3 -#define dst ip -#define tmp2 r10 - -#ifndef USE_NEON -/* For bulk copies using GP registers. */ -#define A_l r2 /* Call-clobbered. */ -#define A_h r3 /* Call-clobbered. */ -#define B_l r4 -#define B_h r5 -#define C_l r6 -#define C_h r7 -#define D_l r8 -#define D_h r9 -#endif - -/* Number of lines ahead to pre-fetch data. If you change this the code - below will need adjustment to compensate. */ - -#define prefetch_lines 5 - -#ifdef USE_VFP - .macro cpy_line_vfp vreg, base - vstr \vreg, [dst, #\base] - vldr \vreg, [src, #\base] - vstr d0, [dst, #\base + 8] - vldr d0, [src, #\base + 8] - vstr d1, [dst, #\base + 16] - vldr d1, [src, #\base + 16] - vstr d2, [dst, #\base + 24] - vldr d2, [src, #\base + 24] - vstr \vreg, [dst, #\base + 32] - vldr \vreg, [src, #\base + prefetch_lines * 64 - 32] - vstr d0, [dst, #\base + 40] - vldr d0, [src, #\base + 40] - vstr d1, [dst, #\base + 48] - vldr d1, [src, #\base + 48] - vstr d2, [dst, #\base + 56] - vldr d2, [src, #\base + 56] - .endm - - .macro cpy_tail_vfp vreg, base - vstr \vreg, [dst, #\base] - vldr \vreg, [src, #\base] - vstr d0, [dst, #\base + 8] - vldr d0, [src, #\base + 8] - vstr d1, [dst, #\base + 16] - vldr d1, [src, #\base + 16] - vstr d2, [dst, #\base + 24] - vldr d2, [src, #\base + 24] - vstr \vreg, [dst, #\base + 32] - vstr d0, [dst, #\base + 40] - vldr d0, [src, #\base + 40] - vstr d1, [dst, #\base + 48] - vldr d1, [src, #\base + 48] - vstr d2, [dst, #\base + 56] - vldr d2, [src, #\base + 56] - .endm -#endif + .text + .syntax unified + .fpu neon ENTRY(__memcpy_chk) - cmp r2, r3 - bhi __memcpy_chk_fail + cmp r2, r3 + bhi __memcpy_chk_fail // Fall through to memcpy... END(__memcpy_chk) ENTRY(memcpy) - .p2align 6 - mov dst, dstin /* Preserve dstin, we need to return it. */ - cmp count, #64 - bge cpy_not_short - /* Deal with small copies quickly by dropping straight into the - exit block. */ - -tail63unaligned: -#ifdef USE_NEON - and tmp1, count, #0x38 - rsb tmp1, tmp1, #(56 - PC_OFFSET + INSN_SIZE) - add pc, pc, tmp1 - vld1.8 {d0}, [src]! /* 14 words to go. */ - vst1.8 {d0}, [dst]! - vld1.8 {d0}, [src]! /* 12 words to go. */ - vst1.8 {d0}, [dst]! - vld1.8 {d0}, [src]! /* 10 words to go. */ - vst1.8 {d0}, [dst]! - vld1.8 {d0}, [src]! /* 8 words to go. */ - vst1.8 {d0}, [dst]! - vld1.8 {d0}, [src]! /* 6 words to go. */ - vst1.8 {d0}, [dst]! - vld1.8 {d0}, [src]! /* 4 words to go. */ - vst1.8 {d0}, [dst]! - vld1.8 {d0}, [src]! /* 2 words to go. */ - vst1.8 {d0}, [dst]! - - tst count, #4 - ldrne tmp1, [src], #4 - strne tmp1, [dst], #4 -#else - /* Copy up to 15 full words of data. May not be aligned. */ - /* Cannot use VFP for unaligned data. */ - and tmp1, count, #0x3c - add dst, dst, tmp1 - add src, src, tmp1 - rsb tmp1, tmp1, #(60 - PC_OFFSET/2 + INSN_SIZE/2) - /* Jump directly into the sequence below at the correct offset. */ - add pc, pc, tmp1, lsl #1 - - ldr tmp1, [src, #-60] /* 15 words to go. */ - str tmp1, [dst, #-60] - - ldr tmp1, [src, #-56] /* 14 words to go. */ - str tmp1, [dst, #-56] - ldr tmp1, [src, #-52] - str tmp1, [dst, #-52] - - ldr tmp1, [src, #-48] /* 12 words to go. */ - str tmp1, [dst, #-48] - ldr tmp1, [src, #-44] - str tmp1, [dst, #-44] - - ldr tmp1, [src, #-40] /* 10 words to go. */ - str tmp1, [dst, #-40] - ldr tmp1, [src, #-36] - str tmp1, [dst, #-36] - - ldr tmp1, [src, #-32] /* 8 words to go. */ - str tmp1, [dst, #-32] - ldr tmp1, [src, #-28] - str tmp1, [dst, #-28] - - ldr tmp1, [src, #-24] /* 6 words to go. */ - str tmp1, [dst, #-24] - ldr tmp1, [src, #-20] - str tmp1, [dst, #-20] - - ldr tmp1, [src, #-16] /* 4 words to go. */ - str tmp1, [dst, #-16] - ldr tmp1, [src, #-12] - str tmp1, [dst, #-12] - - ldr tmp1, [src, #-8] /* 2 words to go. */ - str tmp1, [dst, #-8] - ldr tmp1, [src, #-4] - str tmp1, [dst, #-4] -#endif - - lsls count, count, #31 - ldrhcs tmp1, [src], #2 - ldrbne src, [src] /* Src is dead, use as a scratch. */ - strhcs tmp1, [dst], #2 - strbne src, [dst] - bx lr - -cpy_not_short: - /* At least 64 bytes to copy, but don't know the alignment yet. */ - str tmp2, [sp, #-FRAME_SIZE]! - and tmp2, src, #7 - and tmp1, dst, #7 - cmp tmp1, tmp2 - bne cpy_notaligned - -#ifdef USE_VFP - /* Magic dust alert! Force VFP on Cortex-A9. Experiments show - that the FP pipeline is much better at streaming loads and - stores. This is outside the critical loop. */ - vmov.f32 s0, s0 -#endif - - /* SRC and DST have the same mutual 64-bit alignment, but we may - still need to pre-copy some bytes to get to natural alignment. - We bring SRC and DST into full 64-bit alignment. */ - lsls tmp2, dst, #29 - beq 1f - rsbs tmp2, tmp2, #0 - sub count, count, tmp2, lsr #29 - ldrmi tmp1, [src], #4 - strmi tmp1, [dst], #4 - lsls tmp2, tmp2, #2 - ldrhcs tmp1, [src], #2 - ldrbne tmp2, [src], #1 - strhcs tmp1, [dst], #2 - strbne tmp2, [dst], #1 - -1: - subs tmp2, count, #64 /* Use tmp2 for count. */ - blt tail63aligned - - cmp tmp2, #512 - bge cpy_body_long - -cpy_body_medium: /* Count in tmp2. */ -#ifdef USE_VFP -1: - vldr d0, [src, #0] - subs tmp2, tmp2, #64 - vldr d1, [src, #8] - vstr d0, [dst, #0] - vldr d0, [src, #16] - vstr d1, [dst, #8] - vldr d1, [src, #24] - vstr d0, [dst, #16] - vldr d0, [src, #32] - vstr d1, [dst, #24] - vldr d1, [src, #40] - vstr d0, [dst, #32] - vldr d0, [src, #48] - vstr d1, [dst, #40] - vldr d1, [src, #56] - vstr d0, [dst, #48] - add src, src, #64 - vstr d1, [dst, #56] - add dst, dst, #64 - bge 1b - tst tmp2, #0x3f - beq done - -tail63aligned: /* Count in tmp2. */ - and tmp1, tmp2, #0x38 - add dst, dst, tmp1 - add src, src, tmp1 - rsb tmp1, tmp1, #(56 - PC_OFFSET + INSN_SIZE) - add pc, pc, tmp1 - - vldr d0, [src, #-56] /* 14 words to go. */ - vstr d0, [dst, #-56] - vldr d0, [src, #-48] /* 12 words to go. */ - vstr d0, [dst, #-48] - vldr d0, [src, #-40] /* 10 words to go. */ - vstr d0, [dst, #-40] - vldr d0, [src, #-32] /* 8 words to go. */ - vstr d0, [dst, #-32] - vldr d0, [src, #-24] /* 6 words to go. */ - vstr d0, [dst, #-24] - vldr d0, [src, #-16] /* 4 words to go. */ - vstr d0, [dst, #-16] - vldr d0, [src, #-8] /* 2 words to go. */ - vstr d0, [dst, #-8] -#else - sub src, src, #8 - sub dst, dst, #8 -1: - ldrd A_l, A_h, [src, #8] - strd A_l, A_h, [dst, #8] - ldrd A_l, A_h, [src, #16] - strd A_l, A_h, [dst, #16] - ldrd A_l, A_h, [src, #24] - strd A_l, A_h, [dst, #24] - ldrd A_l, A_h, [src, #32] - strd A_l, A_h, [dst, #32] - ldrd A_l, A_h, [src, #40] - strd A_l, A_h, [dst, #40] - ldrd A_l, A_h, [src, #48] - strd A_l, A_h, [dst, #48] - ldrd A_l, A_h, [src, #56] - strd A_l, A_h, [dst, #56] - ldrd A_l, A_h, [src, #64]! - strd A_l, A_h, [dst, #64]! - subs tmp2, tmp2, #64 - bge 1b - tst tmp2, #0x3f - bne 1f - ldr tmp2,[sp], #FRAME_SIZE - bx lr -1: - add src, src, #8 - add dst, dst, #8 - -tail63aligned: /* Count in tmp2. */ - /* Copy up to 7 d-words of data. Similar to Ltail63unaligned, but - we know that the src and dest are 64-bit aligned so we can use - LDRD/STRD to improve efficiency. */ - /* TMP2 is now negative, but we don't care about that. The bottom - six bits still tell us how many bytes are left to copy. */ - - and tmp1, tmp2, #0x38 - add dst, dst, tmp1 - add src, src, tmp1 - rsb tmp1, tmp1, #(56 - PC_OFFSET + INSN_SIZE) - add pc, pc, tmp1 - ldrd A_l, A_h, [src, #-56] /* 14 words to go. */ - strd A_l, A_h, [dst, #-56] - ldrd A_l, A_h, [src, #-48] /* 12 words to go. */ - strd A_l, A_h, [dst, #-48] - ldrd A_l, A_h, [src, #-40] /* 10 words to go. */ - strd A_l, A_h, [dst, #-40] - ldrd A_l, A_h, [src, #-32] /* 8 words to go. */ - strd A_l, A_h, [dst, #-32] - ldrd A_l, A_h, [src, #-24] /* 6 words to go. */ - strd A_l, A_h, [dst, #-24] - ldrd A_l, A_h, [src, #-16] /* 4 words to go. */ - strd A_l, A_h, [dst, #-16] - ldrd A_l, A_h, [src, #-8] /* 2 words to go. */ - strd A_l, A_h, [dst, #-8] - -#endif - tst tmp2, #4 - ldrne tmp1, [src], #4 - strne tmp1, [dst], #4 - lsls tmp2, tmp2, #31 /* Count (tmp2) now dead. */ - ldrhcs tmp1, [src], #2 - ldrbne tmp2, [src] - strhcs tmp1, [dst], #2 - strbne tmp2, [dst] - -done: - ldr tmp2, [sp], #FRAME_SIZE - bx lr - -cpy_body_long: /* Count in tmp2. */ - - /* Long copy. We know that there's at least (prefetch_lines * 64) - bytes to go. */ -#ifdef USE_VFP - /* Don't use PLD. Instead, read some data in advance of the current - copy position into a register. This should act like a PLD - operation but we won't have to repeat the transfer. */ - - vldr d3, [src, #0] - vldr d4, [src, #64] - vldr d5, [src, #128] - vldr d6, [src, #192] - vldr d7, [src, #256] - - vldr d0, [src, #8] - vldr d1, [src, #16] - vldr d2, [src, #24] - add src, src, #32 - - subs tmp2, tmp2, #prefetch_lines * 64 * 2 - blt 2f -1: - cpy_line_vfp d3, 0 - cpy_line_vfp d4, 64 - cpy_line_vfp d5, 128 - add dst, dst, #3 * 64 - add src, src, #3 * 64 - cpy_line_vfp d6, 0 - cpy_line_vfp d7, 64 - add dst, dst, #2 * 64 - add src, src, #2 * 64 - subs tmp2, tmp2, #prefetch_lines * 64 - bge 1b - -2: - cpy_tail_vfp d3, 0 - cpy_tail_vfp d4, 64 - cpy_tail_vfp d5, 128 - add src, src, #3 * 64 - add dst, dst, #3 * 64 - cpy_tail_vfp d6, 0 - vstr d7, [dst, #64] - vldr d7, [src, #64] - vstr d0, [dst, #64 + 8] - vldr d0, [src, #64 + 8] - vstr d1, [dst, #64 + 16] - vldr d1, [src, #64 + 16] - vstr d2, [dst, #64 + 24] - vldr d2, [src, #64 + 24] - vstr d7, [dst, #64 + 32] - add src, src, #96 - vstr d0, [dst, #64 + 40] - vstr d1, [dst, #64 + 48] - vstr d2, [dst, #64 + 56] - add dst, dst, #128 - add tmp2, tmp2, #prefetch_lines * 64 - b cpy_body_medium -#else - /* Long copy. Use an SMS style loop to maximize the I/O - bandwidth of the core. We don't have enough spare registers - to synthesise prefetching, so use PLD operations. */ - /* Pre-bias src and dst. */ - sub src, src, #8 - sub dst, dst, #8 - pld [src, #8] - pld [src, #72] - subs tmp2, tmp2, #64 - pld [src, #136] - ldrd A_l, A_h, [src, #8] - strd B_l, B_h, [sp, #8] - ldrd B_l, B_h, [src, #16] - strd C_l, C_h, [sp, #16] - ldrd C_l, C_h, [src, #24] - strd D_l, D_h, [sp, #24] - pld [src, #200] - ldrd D_l, D_h, [src, #32]! - b 1f - .p2align 6 -2: - pld [src, #232] - strd A_l, A_h, [dst, #40] - ldrd A_l, A_h, [src, #40] - strd B_l, B_h, [dst, #48] - ldrd B_l, B_h, [src, #48] - strd C_l, C_h, [dst, #56] - ldrd C_l, C_h, [src, #56] - strd D_l, D_h, [dst, #64]! - ldrd D_l, D_h, [src, #64]! - subs tmp2, tmp2, #64 -1: - strd A_l, A_h, [dst, #8] - ldrd A_l, A_h, [src, #8] - strd B_l, B_h, [dst, #16] - ldrd B_l, B_h, [src, #16] - strd C_l, C_h, [dst, #24] - ldrd C_l, C_h, [src, #24] - strd D_l, D_h, [dst, #32] - ldrd D_l, D_h, [src, #32] - bcs 2b - /* Save the remaining bytes and restore the callee-saved regs. */ - strd A_l, A_h, [dst, #40] - add src, src, #40 - strd B_l, B_h, [dst, #48] - ldrd B_l, B_h, [sp, #8] - strd C_l, C_h, [dst, #56] - ldrd C_l, C_h, [sp, #16] - strd D_l, D_h, [dst, #64] - ldrd D_l, D_h, [sp, #24] - add dst, dst, #72 - tst tmp2, #0x3f - bne tail63aligned - ldr tmp2, [sp], #FRAME_SIZE - bx lr -#endif - -cpy_notaligned: - pld [src] - pld [src, #64] - /* There's at least 64 bytes to copy, but there is no mutual - alignment. */ - /* Bring DST to 64-bit alignment. */ - lsls tmp2, dst, #29 - pld [src, #(2 * 64)] - beq 1f - rsbs tmp2, tmp2, #0 - sub count, count, tmp2, lsr #29 - ldrmi tmp1, [src], #4 - strmi tmp1, [dst], #4 - lsls tmp2, tmp2, #2 - ldrbne tmp1, [src], #1 - ldrhcs tmp2, [src], #2 - strbne tmp1, [dst], #1 - strhcs tmp2, [dst], #2 -1: - pld [src, #(3 * 64)] - subs count, count, #64 - ldrmi tmp2, [sp], #FRAME_SIZE - bmi tail63unaligned - pld [src, #(4 * 64)] - -#ifdef USE_NEON - vld1.8 {d0-d3}, [src]! - vld1.8 {d4-d7}, [src]! - subs count, count, #64 - bmi 2f -1: - pld [src, #(4 * 64)] - vst1.8 {d0-d3}, [ALIGN (dst, 64)]! - vld1.8 {d0-d3}, [src]! - vst1.8 {d4-d7}, [ALIGN (dst, 64)]! - vld1.8 {d4-d7}, [src]! - subs count, count, #64 - bpl 1b -2: - vst1.8 {d0-d3}, [ALIGN (dst, 64)]! - vst1.8 {d4-d7}, [ALIGN (dst, 64)]! - ands count, count, #0x3f -#else - /* Use an SMS style loop to maximize the I/O bandwidth. */ - sub src, src, #4 - sub dst, dst, #8 - subs tmp2, count, #64 /* Use tmp2 for count. */ - ldr A_l, [src, #4] - ldr A_h, [src, #8] - strd B_l, B_h, [sp, #8] - ldr B_l, [src, #12] - ldr B_h, [src, #16] - strd C_l, C_h, [sp, #16] - ldr C_l, [src, #20] - ldr C_h, [src, #24] - strd D_l, D_h, [sp, #24] - ldr D_l, [src, #28] - ldr D_h, [src, #32]! - b 1f - .p2align 6 -2: - pld [src, #(5 * 64) - (32 - 4)] - strd A_l, A_h, [dst, #40] - ldr A_l, [src, #36] - ldr A_h, [src, #40] - strd B_l, B_h, [dst, #48] - ldr B_l, [src, #44] - ldr B_h, [src, #48] - strd C_l, C_h, [dst, #56] - ldr C_l, [src, #52] - ldr C_h, [src, #56] - strd D_l, D_h, [dst, #64]! - ldr D_l, [src, #60] - ldr D_h, [src, #64]! - subs tmp2, tmp2, #64 -1: - strd A_l, A_h, [dst, #8] - ldr A_l, [src, #4] - ldr A_h, [src, #8] - strd B_l, B_h, [dst, #16] - ldr B_l, [src, #12] - ldr B_h, [src, #16] - strd C_l, C_h, [dst, #24] - ldr C_l, [src, #20] - ldr C_h, [src, #24] - strd D_l, D_h, [dst, #32] - ldr D_l, [src, #28] - ldr D_h, [src, #32] - bcs 2b - - /* Save the remaining bytes and restore the callee-saved regs. */ - strd A_l, A_h, [dst, #40] - add src, src, #36 - strd B_l, B_h, [dst, #48] - ldrd B_l, B_h, [sp, #8] - strd C_l, C_h, [dst, #56] - ldrd C_l, C_h, [sp, #16] - strd D_l, D_h, [dst, #64] - ldrd D_l, D_h, [sp, #24] - add dst, dst, #72 - ands count, tmp2, #0x3f -#endif - ldr tmp2, [sp], #FRAME_SIZE - bne tail63unaligned - bx lr + pld [r1, #64] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 END(memcpy) #define MEMCPY_BASE __memcpy_base From 02b44628a5ee93ec1cac912b6f10073ec9736c57 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Thu, 23 Jul 2015 20:27:42 -0700 Subject: [PATCH 84/89] Add optimized cortex-a7/cortex-a53 memset/memcpy. Add an optimized memset that is ~20% faster for cortex-a7 and cortex-a53. Add a 32 bit optimized cortex-a53 memcpy that is about ~20% faster on cached data. Fix the cortex-a15 __str{cat,cpy}_chk.S, memcpy_base.S to remove the phony functions, since they aren't needed any more. Then add a direct include of these for cortex-a53. Verified the new functions by stepping through all of the major paths and verifying the backtrace is still correct. Change-Id: I65b13d02a5a780de5e051b0175dbe3bf64895b91 --- .../arch-arm/cortex-a15/bionic/__strcat_chk.S | 192 +--------------- .../cortex-a15/bionic/__strcat_chk_common.S | 212 ++++++++++++++++++ .../arch-arm/cortex-a15/bionic/__strcpy_chk.S | 156 +------------ .../cortex-a15/bionic/__strcpy_chk_common.S | 173 ++++++++++++++ libc/arch-arm/cortex-a15/bionic/memcpy.S | 79 +------ libc/arch-arm/cortex-a15/bionic/memcpy_base.S | 42 ++-- .../cortex-a15/bionic/memcpy_common.S | 103 +++++++++ .../arch-arm/cortex-a53/bionic/__strcat_chk.S | 32 +++ .../arch-arm/cortex-a53/bionic/__strcpy_chk.S | 32 +++ libc/arch-arm/cortex-a53/bionic/memcpy.S | 32 +++ libc/arch-arm/cortex-a53/bionic/memcpy_base.S | 143 ++++++++++++ libc/arch-arm/cortex-a53/cortex-a53.mk | 14 +- libc/arch-arm/cortex-a7/bionic/memset.S | 180 +++++++++++++++ 13 files changed, 958 insertions(+), 432 deletions(-) create mode 100644 libc/arch-arm/cortex-a15/bionic/__strcat_chk_common.S create mode 100644 libc/arch-arm/cortex-a15/bionic/__strcpy_chk_common.S create mode 100644 libc/arch-arm/cortex-a15/bionic/memcpy_common.S create mode 100644 libc/arch-arm/cortex-a53/bionic/__strcat_chk.S create mode 100644 libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S create mode 100644 libc/arch-arm/cortex-a53/bionic/memcpy.S create mode 100644 libc/arch-arm/cortex-a53/bionic/memcpy_base.S create mode 100644 libc/arch-arm/cortex-a7/bionic/memset.S diff --git a/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S index a2e9c229e..3692f0425 100644 --- a/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S +++ b/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 The Android Open Source Project + * Copyright (C) 2015 The Android Open Source Project * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,191 +26,7 @@ * SUCH DAMAGE. */ -#include -#include +// Indicate which memcpy base file to include. +#define MEMCPY_BASE "memcpy_base.S" - .syntax unified - - .thumb - .thumb_func - -// Get the length of src string, then get the source of the dst string. -// Check that the two lengths together don't exceed the threshold, then -// do a memcpy of the data. -ENTRY(__strcat_chk) - pld [r0, #0] - push {r0, lr} - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - push {r4, r5} - .cfi_adjust_cfa_offset 8 - .cfi_rel_offset r4, 0 - .cfi_rel_offset r5, 4 - - mov lr, r2 - - // Save the dst register to r5 - mov r5, r0 - - // Zero out r4 - eor r4, r4, r4 - - // r1 contains the address of the string to count. -.L_strlen_start: - mov r0, r1 - ands r3, r1, #7 - beq .L_mainloop - - // Align to a double word (64 bits). - rsb r3, r3, #8 - lsls ip, r3, #31 - beq .L_align_to_32 - - ldrb r2, [r1], #1 - cbz r2, .L_update_count_and_finish - -.L_align_to_32: - bcc .L_align_to_64 - ands ip, r3, #2 - beq .L_align_to_64 - - ldrb r2, [r1], #1 - cbz r2, .L_update_count_and_finish - ldrb r2, [r1], #1 - cbz r2, .L_update_count_and_finish - -.L_align_to_64: - tst r3, #4 - beq .L_mainloop - ldr r3, [r1], #4 - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne .L_zero_in_second_register - - .p2align 2 -.L_mainloop: - ldrd r2, r3, [r1], #8 - - pld [r1, #64] - - sub ip, r2, #0x01010101 - bic ip, ip, r2 - ands ip, ip, #0x80808080 - bne .L_zero_in_first_register - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne .L_zero_in_second_register - b .L_mainloop - -.L_update_count_and_finish: - sub r3, r1, r0 - sub r3, r3, #1 - b .L_finish - -.L_zero_in_first_register: - sub r3, r1, r0 - lsls r2, ip, #17 - bne .L_sub8_and_finish - bcs .L_sub7_and_finish - lsls ip, ip, #1 - bne .L_sub6_and_finish - - sub r3, r3, #5 - b .L_finish - -.L_sub8_and_finish: - sub r3, r3, #8 - b .L_finish - -.L_sub7_and_finish: - sub r3, r3, #7 - b .L_finish - -.L_sub6_and_finish: - sub r3, r3, #6 - b .L_finish - -.L_zero_in_second_register: - sub r3, r1, r0 - lsls r2, ip, #17 - bne .L_sub4_and_finish - bcs .L_sub3_and_finish - lsls ip, ip, #1 - bne .L_sub2_and_finish - - sub r3, r3, #1 - b .L_finish - -.L_sub4_and_finish: - sub r3, r3, #4 - b .L_finish - -.L_sub3_and_finish: - sub r3, r3, #3 - b .L_finish - -.L_sub2_and_finish: - sub r3, r3, #2 - -.L_finish: - cmp r4, #0 - bne .L_strlen_done - - // Time to get the dst string length. - mov r1, r5 - - // Save the original source address to r5. - mov r5, r0 - - // Save the current length (adding 1 for the terminator). - add r4, r3, #1 - b .L_strlen_start - - // r0 holds the pointer to the dst string. - // r3 holds the dst string length. - // r4 holds the src string length + 1. -.L_strlen_done: - add r2, r3, r4 - cmp r2, lr - bhi __strcat_chk_failed - - // Set up the registers for the memcpy code. - mov r1, r5 - pld [r1, #64] - mov r2, r4 - add r0, r0, r3 - pop {r4, r5} -END(__strcat_chk) - -#define MEMCPY_BASE __strcat_chk_memcpy_base -#define MEMCPY_BASE_ALIGNED __strcat_chk_memcpy_base_aligned - -#include "memcpy_base.S" - -ENTRY_PRIVATE(__strcat_chk_failed) - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - .cfi_adjust_cfa_offset 8 - .cfi_rel_offset r4, 0 - .cfi_rel_offset r5, 4 - - ldr r0, error_message - ldr r1, error_code -1: - add r0, pc - bl __fortify_chk_fail -error_code: - .word BIONIC_EVENT_STRCAT_BUFFER_OVERFLOW -error_message: - .word error_string-(1b+4) -END(__strcat_chk_failed) - - .data -error_string: - .string "strcat: prevented write past end of buffer" +#include "__strcat_chk_common.S" diff --git a/libc/arch-arm/cortex-a15/bionic/__strcat_chk_common.S b/libc/arch-arm/cortex-a15/bionic/__strcat_chk_common.S new file mode 100644 index 000000000..de66967ee --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/__strcat_chk_common.S @@ -0,0 +1,212 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + + .syntax unified + + .thumb + .thumb_func + +// Get the length of src string, then get the source of the dst string. +// Check that the two lengths together don't exceed the threshold, then +// do a memcpy of the data. +ENTRY(__strcat_chk) + pld [r0, #0] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + push {r4, r5} + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 + + mov lr, r2 + + // Save the dst register to r5 + mov r5, r0 + + // Zero out r4 + eor r4, r4, r4 + + // r1 contains the address of the string to count. +.L_strlen_start: + mov r0, r1 + ands r3, r1, #7 + beq .L_mainloop + + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq .L_align_to_32 + + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_32: + bcc .L_align_to_64 + ands ip, r3, #2 + beq .L_align_to_64 + + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_64: + tst r3, #4 + beq .L_mainloop + ldr r3, [r1], #4 + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + + .p2align 2 +.L_mainloop: + ldrd r2, r3, [r1], #8 + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne .L_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + b .L_mainloop + +.L_update_count_and_finish: + sub r3, r1, r0 + sub r3, r3, #1 + b .L_finish + +.L_zero_in_first_register: + sub r3, r1, r0 + lsls r2, ip, #17 + bne .L_sub8_and_finish + bcs .L_sub7_and_finish + lsls ip, ip, #1 + bne .L_sub6_and_finish + + sub r3, r3, #5 + b .L_finish + +.L_sub8_and_finish: + sub r3, r3, #8 + b .L_finish + +.L_sub7_and_finish: + sub r3, r3, #7 + b .L_finish + +.L_sub6_and_finish: + sub r3, r3, #6 + b .L_finish + +.L_zero_in_second_register: + sub r3, r1, r0 + lsls r2, ip, #17 + bne .L_sub4_and_finish + bcs .L_sub3_and_finish + lsls ip, ip, #1 + bne .L_sub2_and_finish + + sub r3, r3, #1 + b .L_finish + +.L_sub4_and_finish: + sub r3, r3, #4 + b .L_finish + +.L_sub3_and_finish: + sub r3, r3, #3 + b .L_finish + +.L_sub2_and_finish: + sub r3, r3, #2 + +.L_finish: + cmp r4, #0 + bne .L_strlen_done + + // Time to get the dst string length. + mov r1, r5 + + // Save the original source address to r5. + mov r5, r0 + + // Save the current length (adding 1 for the terminator). + add r4, r3, #1 + b .L_strlen_start + + // r0 holds the pointer to the dst string. + // r3 holds the dst string length. + // r4 holds the src string length + 1. +.L_strlen_done: + add r2, r3, r4 + cmp r2, lr + bhi .L_strcat_chk_failed + + // Set up the registers for the memcpy code. + mov r1, r5 + pld [r1, #64] + mov r2, r4 + add r0, r0, r3 + pop {r4, r5} + .cfi_adjust_cfa_offset -8 + .cfi_restore r4 + .cfi_restore r5 + +#include MEMCPY_BASE + + // Undo the above cfi directives + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 +.L_strcat_chk_failed: + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_STRCAT_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+4) +END(__strcat_chk) + + .data +error_string: + .string "strcat: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S index db766863a..d8cb3d9d7 100644 --- a/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S +++ b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 The Android Open Source Project + * Copyright (C) 2015 The Android Open Source Project * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,155 +26,7 @@ * SUCH DAMAGE. */ -#include -#include +// Indicate which memcpy base file to include. +#define MEMCPY_BASE "memcpy_base.S" - .syntax unified - - .thumb - .thumb_func - -// Get the length of the source string first, then do a memcpy of the data -// instead of a strcpy. -ENTRY(__strcpy_chk) - pld [r0, #0] - push {r0, lr} - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - - mov lr, r2 - mov r0, r1 - - ands r3, r1, #7 - beq .L_mainloop - - // Align to a double word (64 bits). - rsb r3, r3, #8 - lsls ip, r3, #31 - beq .L_align_to_32 - - ldrb r2, [r0], #1 - cbz r2, .L_update_count_and_finish - -.L_align_to_32: - bcc .L_align_to_64 - ands ip, r3, #2 - beq .L_align_to_64 - - ldrb r2, [r0], #1 - cbz r2, .L_update_count_and_finish - ldrb r2, [r0], #1 - cbz r2, .L_update_count_and_finish - -.L_align_to_64: - tst r3, #4 - beq .L_mainloop - ldr r3, [r0], #4 - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne .L_zero_in_second_register - - .p2align 2 -.L_mainloop: - ldrd r2, r3, [r0], #8 - - pld [r0, #64] - - sub ip, r2, #0x01010101 - bic ip, ip, r2 - ands ip, ip, #0x80808080 - bne .L_zero_in_first_register - - sub ip, r3, #0x01010101 - bic ip, ip, r3 - ands ip, ip, #0x80808080 - bne .L_zero_in_second_register - b .L_mainloop - -.L_update_count_and_finish: - sub r3, r0, r1 - sub r3, r3, #1 - b .L_check_size - -.L_zero_in_first_register: - sub r3, r0, r1 - lsls r2, ip, #17 - bne .L_sub8_and_finish - bcs .L_sub7_and_finish - lsls ip, ip, #1 - bne .L_sub6_and_finish - - sub r3, r3, #5 - b .L_check_size - -.L_sub8_and_finish: - sub r3, r3, #8 - b .L_check_size - -.L_sub7_and_finish: - sub r3, r3, #7 - b .L_check_size - -.L_sub6_and_finish: - sub r3, r3, #6 - b .L_check_size - -.L_zero_in_second_register: - sub r3, r0, r1 - lsls r2, ip, #17 - bne .L_sub4_and_finish - bcs .L_sub3_and_finish - lsls ip, ip, #1 - bne .L_sub2_and_finish - - sub r3, r3, #1 - b .L_check_size - -.L_sub4_and_finish: - sub r3, r3, #4 - b .L_check_size - -.L_sub3_and_finish: - sub r3, r3, #3 - b .L_check_size - -.L_sub2_and_finish: - sub r3, r3, #2 - -.L_check_size: - pld [r1, #0] - pld [r1, #64] - ldr r0, [sp] - cmp r3, lr - bhs __strcpy_chk_failed - - // Add 1 for copy length to get the string terminator. - add r2, r3, #1 -END(__strcpy_chk) - -#define MEMCPY_BASE __strcpy_chk_memcpy_base -#define MEMCPY_BASE_ALIGNED __strcpy_chk_memcpy_base_aligned -#include "memcpy_base.S" - -ENTRY_PRIVATE(__strcpy_chk_failed) - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - - ldr r0, error_message - ldr r1, error_code -1: - add r0, pc - bl __fortify_chk_fail -error_code: - .word BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW -error_message: - .word error_string-(1b+4) -END(__strcpy_chk_failed) - - .data -error_string: - .string "strcpy: prevented write past end of buffer" +#include "__strcpy_chk_common.S" diff --git a/libc/arch-arm/cortex-a15/bionic/__strcpy_chk_common.S b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk_common.S new file mode 100644 index 000000000..69ebcb497 --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk_common.S @@ -0,0 +1,173 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + + .syntax unified + + .thumb + .thumb_func + +// Get the length of the source string first, then do a memcpy of the data +// instead of a strcpy. +ENTRY(__strcpy_chk) + pld [r0, #0] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + + mov lr, r2 + mov r0, r1 + + ands r3, r1, #7 + beq .L_mainloop + + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq .L_align_to_32 + + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_32: + bcc .L_align_to_64 + ands ip, r3, #2 + beq .L_align_to_64 + + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_64: + tst r3, #4 + beq .L_mainloop + ldr r3, [r0], #4 + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + + .p2align 2 +.L_mainloop: + ldrd r2, r3, [r0], #8 + + pld [r0, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne .L_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + b .L_mainloop + +.L_update_count_and_finish: + sub r3, r0, r1 + sub r3, r3, #1 + b .L_check_size + +.L_zero_in_first_register: + sub r3, r0, r1 + lsls r2, ip, #17 + bne .L_sub8_and_finish + bcs .L_sub7_and_finish + lsls ip, ip, #1 + bne .L_sub6_and_finish + + sub r3, r3, #5 + b .L_check_size + +.L_sub8_and_finish: + sub r3, r3, #8 + b .L_check_size + +.L_sub7_and_finish: + sub r3, r3, #7 + b .L_check_size + +.L_sub6_and_finish: + sub r3, r3, #6 + b .L_check_size + +.L_zero_in_second_register: + sub r3, r0, r1 + lsls r2, ip, #17 + bne .L_sub4_and_finish + bcs .L_sub3_and_finish + lsls ip, ip, #1 + bne .L_sub2_and_finish + + sub r3, r3, #1 + b .L_check_size + +.L_sub4_and_finish: + sub r3, r3, #4 + b .L_check_size + +.L_sub3_and_finish: + sub r3, r3, #3 + b .L_check_size + +.L_sub2_and_finish: + sub r3, r3, #2 + +.L_check_size: + pld [r1, #0] + pld [r1, #64] + ldr r0, [sp] + cmp r3, lr + bhs .L_strcpy_chk_failed + + // Add 1 for copy length to get the string terminator. + add r2, r3, #1 + +#include MEMCPY_BASE + +.L_strcpy_chk_failed: + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+4) +END(__strcpy_chk) + + .data +error_string: + .string "strcpy: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy.S b/libc/arch-arm/cortex-a15/bionic/memcpy.S index 410b663e2..537f3de2f 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy.S @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008 The Android Open Source Project + * Copyright (C) 2015 The Android Open Source Project * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -25,79 +25,8 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ -/* - * Copyright (c) 2013 ARM Ltd - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the company may not be used to endorse or promote - * products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -// Prototype: void *memcpy (void *dst, const void *src, size_t count). - -#include -#include - - .text - .syntax unified - .fpu neon - -ENTRY(__memcpy_chk) - cmp r2, r3 - bhi __memcpy_chk_fail - - // Fall through to memcpy... -END(__memcpy_chk) - -ENTRY(memcpy) - pld [r1, #64] - push {r0, lr} - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 -END(memcpy) - -#define MEMCPY_BASE __memcpy_base -#define MEMCPY_BASE_ALIGNED __memcpy_base_aligned -#include "memcpy_base.S" - -ENTRY_PRIVATE(__memcpy_chk_fail) - // Preserve lr for backtrace. - push {lr} - .cfi_def_cfa_offset 4 - .cfi_rel_offset lr, 0 - ldr r0, error_message - ldr r1, error_code -1: - add r0, pc - bl __fortify_chk_fail -error_code: - .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW -error_message: - .word error_string-(1b+8) -END(__memcpy_chk_fail) +// Indicate which memcpy base file to include. +#define MEMCPY_BASE "memcpy_base.S" - .data -error_string: - .string "memcpy: prevented write past end of buffer" +#include "memcpy_common.S" diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy_base.S b/libc/arch-arm/cortex-a15/bionic/memcpy_base.S index 2a7385247..aac737df5 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy_base.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy_base.S @@ -53,11 +53,7 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -ENTRY_PRIVATE(MEMCPY_BASE) - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 - +.L_memcpy_base: // Assumes that n >= 0, and dst, src are valid pointers. // For any sizes less than 832 use the neon code that doesn't // care about the src alignment. This avoids any checks @@ -168,12 +164,6 @@ ENTRY_PRIVATE(MEMCPY_BASE) eor r3, r0, r1 ands r3, r3, #0x3 bne .L_copy_unknown_alignment -END(MEMCPY_BASE) - -ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED) - .cfi_def_cfa_offset 8 - .cfi_rel_offset r0, 0 - .cfi_rel_offset lr, 4 // To try and improve performance, stack layout changed, // i.e., not keeping the stack looking like users expect @@ -185,7 +175,7 @@ ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED) strd r6, r7, [sp, #-8]! .cfi_adjust_cfa_offset 8 .cfi_rel_offset r6, 0 - .cfi_rel_offset r7, 0 + .cfi_rel_offset r7, 4 strd r8, r9, [sp, #-8]! .cfi_adjust_cfa_offset 8 .cfi_rel_offset r8, 0 @@ -291,10 +281,28 @@ ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED) // Restore registers: optimized pop {r0, pc} ldrd r8, r9, [sp], #8 + .cfi_adjust_cfa_offset -8 + .cfi_restore r8 + .cfi_restore r9 ldrd r6, r7, [sp], #8 + .cfi_adjust_cfa_offset -8 + .cfi_restore r6 + .cfi_restore r7 ldrd r4, r5, [sp], #8 + .cfi_adjust_cfa_offset -8 + .cfi_restore r4 + .cfi_restore r5 pop {r0, pc} + // Put the cfi directives back for the below instructions. + .cfi_adjust_cfa_offset 24 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 + .cfi_rel_offset r6, 8 + .cfi_rel_offset r7, 12 + .cfi_rel_offset r8, 16 + .cfi_rel_offset r9, 20 + .L_dst_not_word_aligned: // Align dst to word. rsb ip, ip, #4 @@ -315,4 +323,12 @@ ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED) // Src is guaranteed to be at least word aligned by this point. b .L_word_aligned -END(MEMCPY_BASE_ALIGNED) + + // Undo any cfi directives from above. + .cfi_adjust_cfa_offset -24 + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 + .cfi_restore r8 + .cfi_restore r9 diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy_common.S b/libc/arch-arm/cortex-a15/bionic/memcpy_common.S new file mode 100644 index 000000000..464fb46b3 --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/memcpy_common.S @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + + .text + .syntax unified + .fpu neon + +ENTRY(__memcpy_chk) + cmp r2, r3 + bhi .L_memcpy_chk_fail + + // Fall through to memcpy... +END(__memcpy_chk) + +// Prototype: void *memcpy (void *dst, const void *src, size_t count). +ENTRY(memcpy) + pld [r1, #64] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + +#include MEMCPY_BASE + + // Undo the cfi instructions from above. + .cfi_def_cfa_offset 0 + .cfi_restore r0 + .cfi_restore lr +.L_memcpy_chk_fail: + // Preserve lr for backtrace. + push {lr} + .cfi_adjust_cfa_offset 4 + .cfi_rel_offset lr, 0 + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) +END(memcpy) + + .data +error_string: + .string "memcpy: prevented write past end of buffer" diff --git a/libc/arch-arm/cortex-a53/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a53/bionic/__strcat_chk.S new file mode 100644 index 000000000..c5bc98a72 --- /dev/null +++ b/libc/arch-arm/cortex-a53/bionic/__strcat_chk.S @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +// Indicate which memcpy base file to include. +#define MEMCPY_BASE "arch-arm/cortex-a53/bionic/memcpy_base.S" + +#include "arch-arm/cortex-a15/bionic/__strcat_chk_common.S" diff --git a/libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S new file mode 100644 index 000000000..1f8945d9f --- /dev/null +++ b/libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +// Indicate which memcpy base file to include. +#define MEMCPY_BASE "arch-arm/cortex-a53/bionic/memcpy_base.S" + +#include "arch-arm/cortex-a15/bionic/__strcpy_chk_common.S" diff --git a/libc/arch-arm/cortex-a53/bionic/memcpy.S b/libc/arch-arm/cortex-a53/bionic/memcpy.S new file mode 100644 index 000000000..664f57442 --- /dev/null +++ b/libc/arch-arm/cortex-a53/bionic/memcpy.S @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +// Indicate which memcpy base file to include. +#define MEMCPY_BASE "arch-arm/cortex-a53/bionic/memcpy_base.S" + +#include "arch-arm/cortex-a15/bionic/memcpy_common.S" diff --git a/libc/arch-arm/cortex-a53/bionic/memcpy_base.S b/libc/arch-arm/cortex-a53/bionic/memcpy_base.S new file mode 100644 index 000000000..2749fc83d --- /dev/null +++ b/libc/arch-arm/cortex-a53/bionic/memcpy_base.S @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +.L_memcpy_base: + // Assumes that n >= 0, and dst, src are valid pointers. + cmp r2, #16 + blo .L_copy_less_than_16_unknown_align + +.L_copy_unknown_alignment: + // Unknown alignment of src and dst. + // Assumes that the first few bytes have already been prefetched. + + // Align destination to 128 bits. The mainloop store instructions + // require this alignment or they will throw an exception. + rsb r3, r0, #0 + ands r3, r3, #0xF + beq 2f + + // Copy up to 15 bytes (count in r3). + sub r2, r2, r3 + movs ip, r3, lsl #31 + + itt mi + ldrbmi lr, [r1], #1 + strbmi lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1], #1 + strbcs ip, [r0], #1 + strbcs lr, [r0], #1 + + movs ip, r3, lsl #29 + bge 1f + // Copies 4 bytes, dst 32 bits aligned before, at least 64 bits after. + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! +1: bcc 2f + // Copies 8 bytes, dst 64 bits aligned before, at least 128 bits after. + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0, :64]! + +2: // Make sure we have at least 64 bytes to copy. + subs r2, r2, #64 + blo 2f + +1: // The main loop copies 64 bytes at a time. + vld1.8 {d0 - d3}, [r1]! + vld1.8 {d4 - d7}, [r1]! + subs r2, r2, #64 + vstmia r0!, {d0 - d7} + pld [r1, #(64*10)] + bhs 1b + +2: // Fix-up the remaining count and make sure we have >= 32 bytes left. + adds r2, r2, #32 + blo 3f + + // 32 bytes. These cache lines were already preloaded. + vld1.8 {d0 - d3}, [r1]! + sub r2, r2, #32 + vst1.8 {d0 - d3}, [r0, :128]! +3: // Less than 32 left. + add r2, r2, #32 + tst r2, #0x10 + beq .L_copy_less_than_16_unknown_align + // Copies 16 bytes, destination 128 bits aligned. + vld1.8 {d0, d1}, [r1]! + vst1.8 {d0, d1}, [r0, :128]! + +.L_copy_less_than_16_unknown_align: + // Copy up to 15 bytes (count in r2). + movs ip, r2, lsl #29 + bcc 1f + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0]! +1: bge 2f + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! + +2: // Copy 0 to 4 bytes. + lsls r2, r2, #31 + itt ne + ldrbne lr, [r1], #1 + strbne lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1] + strbcs ip, [r0], #1 + strbcs lr, [r0] + + pop {r0, pc} diff --git a/libc/arch-arm/cortex-a53/cortex-a53.mk b/libc/arch-arm/cortex-a53/cortex-a53.mk index e6fc66d6e..f61263154 100644 --- a/libc/arch-arm/cortex-a53/cortex-a53.mk +++ b/libc/arch-arm/cortex-a53/cortex-a53.mk @@ -1,12 +1,18 @@ libc_bionic_src_files_arm += \ - arch-arm/cortex-a15/bionic/memchr.S \ - arch-arm/cortex-a15/bionic/memcpy.S \ + arch-arm/cortex-a53/bionic/memcpy.S \ + arch-arm/cortex-a53/bionic/__strcat_chk.S \ + arch-arm/cortex-a53/bionic/__strcpy_chk.S \ + +libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/memset.S \ + +libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/strcat.S \ arch-arm/cortex-a15/bionic/strcmp.S \ arch-arm/cortex-a15/bionic/strcpy.S \ arch-arm/cortex-a15/bionic/strlen.S \ arch-arm/cortex-a15/bionic/stpcpy.S \ - arch-arm/cortex-a15/bionic/__strcat_chk.S \ - arch-arm/cortex-a15/bionic/__strcpy_chk.S \ arch-arm/krait/bionic/memmove.S + +libc_bionic_src_files_arm += \ + arch-arm/cortex-a15/bionic/memchr.S diff --git a/libc/arch-arm/cortex-a7/bionic/memset.S b/libc/arch-arm/cortex-a7/bionic/memset.S new file mode 100644 index 000000000..6365b066a --- /dev/null +++ b/libc/arch-arm/cortex-a7/bionic/memset.S @@ -0,0 +1,180 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include + + /* + * Optimized memset() for ARM. + * + * memset() returns its first argument. + */ + + .fpu neon + .syntax unified + +ENTRY(__memset_chk) + cmp r2, r3 + bls .L_done + + // Preserve lr for backtrace. + push {lr} + .cfi_def_cfa_offset 4 + .cfi_rel_offset lr, 0 + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMSET_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) +END(__memset_chk) + +ENTRY(bzero) + mov r2, r1 + mov r1, #0 +.L_done: + // Fall through to memset... +END(bzero) + +ENTRY(memset) + mov r3, r0 + // At this point only d0, d1 are going to be used below. + vdup.8 q0, r1 + cmp r2, #16 + blo .L_set_less_than_16_unknown_align + +.L_check_alignment: + // Align destination to a double word to avoid the store crossing + // a cache line boundary. + ands ip, r3, #7 + bne .L_do_double_word_align + +.L_double_word_aligned: + // Duplicate since the less than 64 can use d2, d3. + vmov q1, q0 + subs r2, #64 + blo .L_set_less_than_64 + + // Duplicate the copy value so that we can store 64 bytes at a time. + vmov q2, q0 + vmov q3, q0 + +1: // Main loop stores 64 bytes at a time. + subs r2, #64 + vstmia r3!, {d0 - d7} + bge 1b + +.L_set_less_than_64: + // Restore r2 to the count of bytes left to set. + add r2, #64 + lsls ip, r2, #27 + bcc .L_set_less_than_32 + // Set 32 bytes. + vstmia r3!, {d0 - d3} + +.L_set_less_than_32: + bpl .L_set_less_than_16 + // Set 16 bytes. + vstmia r3!, {d0, d1} + +.L_set_less_than_16: + // Less than 16 bytes to set. + lsls ip, r2, #29 + bcc .L_set_less_than_8 + + // Set 8 bytes. + vstmia r3!, {d0} + +.L_set_less_than_8: + bpl .L_set_less_than_4 + // Set 4 bytes + vst1.32 {d0[0]}, [r3]! + +.L_set_less_than_4: + lsls ip, r2, #31 + it ne + strbne r1, [r3], #1 + itt cs + strbcs r1, [r3], #1 + strbcs r1, [r3] + bx lr + +.L_do_double_word_align: + rsb ip, ip, #8 + sub r2, r2, ip + + // Do this comparison now, otherwise we'll need to save a + // register to the stack since we've used all available + // registers. + cmp ip, #4 + blo 1f + + // Need to do a four byte copy. + movs ip, ip, lsl #31 + it mi + strbmi r1, [r3], #1 + itt cs + strbcs r1, [r3], #1 + strbcs r1, [r3], #1 + vst1.32 {d0[0]}, [r3]! + b .L_double_word_aligned + +1: + // No four byte copy. + movs ip, ip, lsl #31 + it mi + strbmi r1, [r3], #1 + itt cs + strbcs r1, [r3], #1 + strbcs r1, [r3], #1 + b .L_double_word_aligned + +.L_set_less_than_16_unknown_align: + // Set up to 15 bytes. + movs ip, r2, lsl #29 + bcc 1f + vst1.8 {d0}, [r3]! +1: bge 2f + vst1.32 {d0[0]}, [r3]! +2: movs ip, r2, lsl #31 + it mi + strbmi r1, [r3], #1 + itt cs + strbcs r1, [r3], #1 + strbcs r1, [r3], #1 + bx lr +END(memset) + + .data +error_string: + .string "memset: prevented write past end of buffer" From 2e5e2d028a64f50cf88ef0a0585d26164e379924 Mon Sep 17 00:00:00 2001 From: Chih-Hung Hsieh Date: Mon, 11 May 2015 11:21:19 -0700 Subject: [PATCH 85/89] Use unified syntax to compile with both llvm and gcc. All arch-arm and arch-arm64 .S files were compiled by gcc with and without this patch. The output object files were identical. When compiled with llvm and this patch, the output files were also identical to gcc's output. BUG: 18061004 Change-Id: Ie26b752887bedb767259f06f7dcac284104b5ba1 --- libc/Android.mk | 5 -- libc/arch-arm/bionic/memcmp.S | 24 +++++----- libc/arch-arm/cortex-a9/bionic/memset.S | 29 ++++++------ libc/arch-arm/generic/bionic/memcpy.S | 62 +++++++++++++------------ libc/arch-arm/generic/bionic/memset.S | 24 +++++----- libc/arch-arm/generic/bionic/strcpy.S | 10 ++-- libc/arch-arm/krait/bionic/memset.S | 7 +-- libc/arch-arm64/generic/bionic/memchr.S | 6 +-- libc/arch-arm64/generic/bionic/strchr.S | 6 +-- 9 files changed, 89 insertions(+), 84 deletions(-) diff --git a/libc/Android.mk b/libc/Android.mk index fb99c81d1..f9bb5b416 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -821,11 +821,6 @@ LOCAL_SRC_FILES := $(libc_bionic_src_files) LOCAL_CFLAGS := $(libc_common_cflags) \ -Wframe-larger-than=2048 \ -ifeq ($(TARGET_ARCH),x86_64) - # Clang assembler has problem with ssse3-strcmp-slm.S, http://b/17302991 - LOCAL_CLANG_ASFLAGS += -no-integrated-as -endif - LOCAL_CONLYFLAGS := $(libc_common_conlyflags) LOCAL_CPPFLAGS := $(libc_common_cppflags) LOCAL_C_INCLUDES := $(libc_common_c_includes) diff --git a/libc/arch-arm/bionic/memcmp.S b/libc/arch-arm/bionic/memcmp.S index 70a2a58ce..c78dbd4bf 100644 --- a/libc/arch-arm/bionic/memcmp.S +++ b/libc/arch-arm/bionic/memcmp.S @@ -40,6 +40,8 @@ * Optimized memcmp() for Cortex-A9. */ +.syntax unified + ENTRY(memcmp) pld [r0, #(CACHE_LINE_SIZE * 0)] pld [r0, #(CACHE_LINE_SIZE * 1)] @@ -161,25 +163,25 @@ ENTRY(memcmp) eors r0, r0, ip ldreq r0, [r4], #4 ldreq ip, [r1, #4]! - eoreqs r0, r0, lr + eorseq r0, r0, lr ldreq r0, [r4], #4 ldreq lr, [r1, #4]! - eoreqs r0, r0, ip + eorseq r0, r0, ip ldreq r0, [r4], #4 ldreq ip, [r1, #4]! - eoreqs r0, r0, lr + eorseq r0, r0, lr ldreq r0, [r4], #4 ldreq lr, [r1, #4]! - eoreqs r0, r0, ip + eorseq r0, r0, ip ldreq r0, [r4], #4 ldreq ip, [r1, #4]! - eoreqs r0, r0, lr + eorseq r0, r0, lr ldreq r0, [r4], #4 ldreq lr, [r1, #4]! - eoreqs r0, r0, ip + eorseq r0, r0, ip ldreq r0, [r4], #4 ldreq ip, [r1, #4]! - eoreqs r0, r0, lr + eorseq r0, r0, lr bne 2f subs r2, r2, #32 bhs 0b @@ -263,17 +265,17 @@ ENTRY(memcmp) ldreq lr, [r1], #4 ldreq r0, [r4], #4 orreq ip, ip, lr, lsl #16 - eoreqs r0, r0, ip + eorseq r0, r0, ip moveq ip, lr, lsr #16 ldreq lr, [r1], #4 ldreq r0, [r4], #4 orreq ip, ip, lr, lsl #16 - eoreqs r0, r0, ip + eorseq r0, r0, ip moveq ip, lr, lsr #16 ldreq lr, [r1], #4 ldreq r0, [r4], #4 orreq ip, ip, lr, lsl #16 - eoreqs r0, r0, ip + eorseq r0, r0, ip bne 7f subs r2, r2, #16 bhs 6b @@ -317,7 +319,7 @@ ENTRY(memcmp) ldreq r7, [r1], #4 ldreq r0, [r4], #4 orreq ip, ip, r7, lsl r6 - eoreqs r0, r0, ip + eorseq r0, r0, ip bne 7f subs r2, r2, #8 bhs 6b diff --git a/libc/arch-arm/cortex-a9/bionic/memset.S b/libc/arch-arm/cortex-a9/bionic/memset.S index 299f5a2d2..8ee6ac26c 100644 --- a/libc/arch-arm/cortex-a9/bionic/memset.S +++ b/libc/arch-arm/cortex-a9/bionic/memset.S @@ -35,6 +35,7 @@ */ .fpu neon + .syntax unified ENTRY(__memset_chk) cmp r2, r3 @@ -100,9 +101,9 @@ ENTRY(memset) 1: bge 2f vst1.32 {d0[0]}, [r0]! 2: movs ip, r2, lsl #31 - strmib r1, [r0], #1 - strcsb r1, [r0], #1 - strcsb r1, [r0], #1 + strbmi r1, [r0], #1 + strbcs r1, [r0], #1 + strbcs r1, [r0], #1 ldmfd sp!, {r0} bx lr END(memset) @@ -131,11 +132,11 @@ ENTRY_PRIVATE(__memset_large_copy) orr r1, r1, r1, lsr #16 movs r12, r3, lsl #31 - strcsb r1, [r0], #1 /* can't use strh (alignment unknown) */ - strcsb r1, [r0], #1 - strmib r1, [r0], #1 + strbcs r1, [r0], #1 /* can't use strh (alignment unknown) */ + strbcs r1, [r0], #1 + strbmi r1, [r0], #1 subs r2, r2, r3 - ldmlsfd sp!, {r0, r4-r7, lr} /* return */ + popls {r0, r4-r7, lr} /* return */ bxls lr /* align the destination to a cache-line */ @@ -155,9 +156,9 @@ ENTRY_PRIVATE(__memset_large_copy) /* conditionally writes 0 to 7 words (length in r3) */ movs r3, r3, lsl #28 - stmcsia r0!, {r1, lr} - stmcsia r0!, {r1, lr} - stmmiia r0!, {r1, lr} + stmcs r0!, {r1, lr} + stmcs r0!, {r1, lr} + stmmi r0!, {r1, lr} movs r3, r3, lsl #2 strcs r1, [r0], #4 @@ -172,13 +173,13 @@ ENTRY_PRIVATE(__memset_large_copy) /* conditionally stores 0 to 31 bytes */ movs r2, r2, lsl #28 - stmcsia r0!, {r1,r3,r12,lr} - stmmiia r0!, {r1, lr} + stmcs r0!, {r1,r3,r12,lr} + stmmi r0!, {r1, lr} movs r2, r2, lsl #2 strcs r1, [r0], #4 - strmih r1, [r0], #2 + strhmi r1, [r0], #2 movs r2, r2, lsl #2 - strcsb r1, [r0] + strbcs r1, [r0] ldmfd sp!, {r0, r4-r7, lr} bx lr END(__memset_large_copy) diff --git a/libc/arch-arm/generic/bionic/memcpy.S b/libc/arch-arm/generic/bionic/memcpy.S index b0c79abf7..ea5a3998b 100644 --- a/libc/arch-arm/generic/bionic/memcpy.S +++ b/libc/arch-arm/generic/bionic/memcpy.S @@ -37,6 +37,8 @@ * so we have to preserve R0. */ + .syntax unified + ENTRY(__memcpy_chk) cmp r2, r3 bhi __memcpy_chk_fail @@ -81,12 +83,12 @@ ENTRY(memcpy) */ movs r12, r3, lsl #31 sub r2, r2, r3 /* we know that r3 <= r2 because r2 >= 4 */ - ldrmib r3, [r1], #1 - ldrcsb r4, [r1], #1 - ldrcsb r12,[r1], #1 - strmib r3, [r0], #1 - strcsb r4, [r0], #1 - strcsb r12,[r0], #1 + ldrbmi r3, [r1], #1 + ldrbcs r4, [r1], #1 + ldrbcs r12,[r1], #1 + strbmi r3, [r0], #1 + strbcs r4, [r0], #1 + strbcs r12,[r0], #1 .Lsrc_aligned: @@ -109,10 +111,10 @@ ENTRY(memcpy) /* conditionally copies 0 to 7 words (length in r3) */ movs r12, r3, lsl #28 - ldmcsia r1!, {r4, r5, r6, r7} /* 16 bytes */ - ldmmiia r1!, {r8, r9} /* 8 bytes */ - stmcsia r0!, {r4, r5, r6, r7} - stmmiia r0!, {r8, r9} + ldmcs r1!, {r4, r5, r6, r7} /* 16 bytes */ + ldmmi r1!, {r8, r9} /* 8 bytes */ + stmcs r0!, {r4, r5, r6, r7} + stmmi r0!, {r8, r9} tst r3, #0x4 ldrne r10,[r1], #4 /* 4 bytes */ strne r10,[r0], #4 @@ -177,18 +179,18 @@ ENTRY(memcpy) /* conditionnaly copies 0 to 31 bytes */ movs r12, r2, lsl #28 - ldmcsia r1!, {r4, r5, r6, r7} /* 16 bytes */ - ldmmiia r1!, {r8, r9} /* 8 bytes */ - stmcsia r0!, {r4, r5, r6, r7} - stmmiia r0!, {r8, r9} + ldmcs r1!, {r4, r5, r6, r7} /* 16 bytes */ + ldmmi r1!, {r8, r9} /* 8 bytes */ + stmcs r0!, {r4, r5, r6, r7} + stmmi r0!, {r8, r9} movs r12, r2, lsl #30 ldrcs r3, [r1], #4 /* 4 bytes */ - ldrmih r4, [r1], #2 /* 2 bytes */ + ldrhmi r4, [r1], #2 /* 2 bytes */ strcs r3, [r0], #4 - strmih r4, [r0], #2 + strhmi r4, [r0], #2 tst r2, #0x1 - ldrneb r3, [r1] /* last byte */ - strneb r3, [r0] + ldrbne r3, [r1] /* last byte */ + strbne r3, [r0] /* we're done! restore everything and return */ 1: ldmfd sp!, {r5-r11} @@ -228,11 +230,11 @@ ENTRY(memcpy) * becomes aligned to 32 bits (r5 = nb of words to copy for alignment) */ movs r5, r5, lsl #31 - strmib r3, [r0], #1 + strbmi r3, [r0], #1 movmi r3, r3, lsr #8 - strcsb r3, [r0], #1 + strbcs r3, [r0], #1 movcs r3, r3, lsr #8 - strcsb r3, [r0], #1 + strbcs r3, [r0], #1 movcs r3, r3, lsr #8 cmp r2, #4 @@ -363,23 +365,23 @@ ENTRY(memcpy) .Lpartial_word_tail: /* we have a partial word in the input buffer */ movs r5, lr, lsl #(31-3) - strmib r3, [r0], #1 + strbmi r3, [r0], #1 movmi r3, r3, lsr #8 - strcsb r3, [r0], #1 + strbcs r3, [r0], #1 movcs r3, r3, lsr #8 - strcsb r3, [r0], #1 + strbcs r3, [r0], #1 /* Refill spilled registers from the stack. Don't update sp. */ ldmfd sp, {r5-r11} .Lcopy_last_3_and_return: movs r2, r2, lsl #31 /* copy remaining 0, 1, 2 or 3 bytes */ - ldrmib r2, [r1], #1 - ldrcsb r3, [r1], #1 - ldrcsb r12,[r1] - strmib r2, [r0], #1 - strcsb r3, [r0], #1 - strcsb r12,[r0] + ldrbmi r2, [r1], #1 + ldrbcs r3, [r1], #1 + ldrbcs r12,[r1] + strbmi r2, [r0], #1 + strbcs r3, [r0], #1 + strbcs r12,[r0] /* we're done! restore sp and spilled registers and return */ add sp, sp, #28 diff --git a/libc/arch-arm/generic/bionic/memset.S b/libc/arch-arm/generic/bionic/memset.S index be35de9ff..d17a9c4e0 100644 --- a/libc/arch-arm/generic/bionic/memset.S +++ b/libc/arch-arm/generic/bionic/memset.S @@ -35,6 +35,8 @@ * memset() returns its first argument. */ + .syntax unified + ENTRY(__memset_chk) cmp r2, r3 bls done @@ -76,11 +78,11 @@ ENTRY(memset) orr r1, r1, r1, lsr #16 movs r12, r3, lsl #31 - strcsb r1, [r0], #1 /* can't use strh (alignment unknown) */ - strcsb r1, [r0], #1 - strmib r1, [r0], #1 + strbcs r1, [r0], #1 /* can't use strh (alignment unknown) */ + strbcs r1, [r0], #1 + strbmi r1, [r0], #1 subs r2, r2, r3 - ldmlsfd sp!, {r0, r4-r7, lr} /* return */ + popls {r0, r4-r7, lr} /* return */ bxls lr /* align the destination to a cache-line */ @@ -100,9 +102,9 @@ ENTRY(memset) /* conditionally writes 0 to 7 words (length in r3) */ movs r3, r3, lsl #28 - stmcsia r0!, {r1, lr} - stmcsia r0!, {r1, lr} - stmmiia r0!, {r1, lr} + stmcs r0!, {r1, lr} + stmcs r0!, {r1, lr} + stmmi r0!, {r1, lr} movs r3, r3, lsl #2 strcs r1, [r0], #4 @@ -117,13 +119,13 @@ ENTRY(memset) /* conditionally stores 0 to 31 bytes */ movs r2, r2, lsl #28 - stmcsia r0!, {r1,r3,r12,lr} - stmmiia r0!, {r1, lr} + stmcs r0!, {r1,r3,r12,lr} + stmmi r0!, {r1, lr} movs r2, r2, lsl #2 strcs r1, [r0], #4 - strmih r1, [r0], #2 + strhmi r1, [r0], #2 movs r2, r2, lsl #2 - strcsb r1, [r0] + strbcs r1, [r0] ldmfd sp!, {r0, r4-r7, lr} bx lr END(memset) diff --git a/libc/arch-arm/generic/bionic/strcpy.S b/libc/arch-arm/generic/bionic/strcpy.S index 802a62da8..89ea098ad 100644 --- a/libc/arch-arm/generic/bionic/strcpy.S +++ b/libc/arch-arm/generic/bionic/strcpy.S @@ -32,6 +32,8 @@ #include #include +.syntax unified + ENTRY(strcpy) pld [r1, #0] eor r2, r0, r1 @@ -108,15 +110,15 @@ ENTRY(strcpy) #ifdef __ARMEB__ tst r2, #0xff00 iteet ne - strneh r2, [ip], #2 + strhne r2, [ip], #2 lsreq r2, r2, #8 - streqb r2, [ip] + strbeq r2, [ip] tstne r2, #0xff #else tst r2, #0xff itet ne - strneh r2, [ip], #2 - streqb r2, [ip] + strhne r2, [ip], #2 + strbeq r2, [ip] tstne r2, #0xff00 #endif bne 5b diff --git a/libc/arch-arm/krait/bionic/memset.S b/libc/arch-arm/krait/bionic/memset.S index e9f643133..a4fbe178f 100644 --- a/libc/arch-arm/krait/bionic/memset.S +++ b/libc/arch-arm/krait/bionic/memset.S @@ -37,6 +37,7 @@ */ .fpu neon + .syntax unified ENTRY(__memset_chk) cmp r2, r3 @@ -98,9 +99,9 @@ ENTRY(memset) 1: bge 2f vst1.32 {d0[0]}, [r0]! 2: movs ip, r2, lsl #31 - strmib r1, [r0], #1 - strcsb r1, [r0], #1 - strcsb r1, [r0], #1 + strbmi r1, [r0], #1 + strbcs r1, [r0], #1 + strbcs r1, [r0], #1 ldmfd sp!, {r0} bx lr END(memset) diff --git a/libc/arch-arm64/generic/bionic/memchr.S b/libc/arch-arm64/generic/bionic/memchr.S index e5ea57d8c..a00dd8dee 100644 --- a/libc/arch-arm64/generic/bionic/memchr.S +++ b/libc/arch-arm64/generic/bionic/memchr.S @@ -101,7 +101,7 @@ ENTRY(memchr) and vhas_chr2.16b, vhas_chr2.16b, vrepmask.16b addp vend.16b, vhas_chr1.16b, vhas_chr2.16b /* 256->128 */ addp vend.16b, vend.16b, vend.16b /* 128->64 */ - mov synd, vend.2d[0] + mov synd, vend.d[0] /* Clear the soff*2 lower bits */ lsl tmp, soff, #1 lsr synd, synd, tmp @@ -121,7 +121,7 @@ ENTRY(memchr) /* Use a fast check for the termination condition */ orr vend.16b, vhas_chr1.16b, vhas_chr2.16b addp vend.2d, vend.2d, vend.2d - mov synd, vend.2d[0] + mov synd, vend.d[0] /* We're not out of data, loop if we haven't found the character */ cbz synd, .Lloop @@ -131,7 +131,7 @@ ENTRY(memchr) and vhas_chr2.16b, vhas_chr2.16b, vrepmask.16b addp vend.16b, vhas_chr1.16b, vhas_chr2.16b /* 256->128 */ addp vend.16b, vend.16b, vend.16b /* 128->64 */ - mov synd, vend.2d[0] + mov synd, vend.d[0] /* Only do the clear for the last possible block */ b.hi .Ltail diff --git a/libc/arch-arm64/generic/bionic/strchr.S b/libc/arch-arm64/generic/bionic/strchr.S index 469b83c63..b54106d93 100644 --- a/libc/arch-arm64/generic/bionic/strchr.S +++ b/libc/arch-arm64/generic/bionic/strchr.S @@ -109,7 +109,7 @@ ENTRY(strchr) addp vend1.16b, vend1.16b, vend2.16b // 128->64 lsr tmp1, tmp3, tmp1 - mov tmp3, vend1.2d[0] + mov tmp3, vend1.d[0] bic tmp1, tmp3, tmp1 // Mask padding bits. cbnz tmp1, .Ltail @@ -124,7 +124,7 @@ ENTRY(strchr) orr vend2.16b, vhas_nul2.16b, vhas_chr2.16b orr vend1.16b, vend1.16b, vend2.16b addp vend1.2d, vend1.2d, vend1.2d - mov tmp1, vend1.2d[0] + mov tmp1, vend1.d[0] cbz tmp1, .Lloop /* Termination condition found. Now need to establish exactly why @@ -138,7 +138,7 @@ ENTRY(strchr) addp vend1.16b, vend1.16b, vend2.16b // 256->128 addp vend1.16b, vend1.16b, vend2.16b // 128->64 - mov tmp1, vend1.2d[0] + mov tmp1, vend1.d[0] .Ltail: /* Count the trailing zeros, by bit reversing... */ rbit tmp1, tmp1 From 22ee2dccdcf62e65a4e6f64266ab622b8479ccc2 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Mon, 6 Jul 2015 12:03:40 -0700 Subject: [PATCH 86/89] Replace bx lr with update of pc from the stack. When there is arm assembler of this format: ldmxx sp!, {..., lr} or pop {..., lr} bx lr It can be replaced with: ldmxx sp!, {..., pc} or pop {..., pc} Change-Id: I928b2d0467b30b5b2eb27ea6a606f5954546853e --- libc/arch-arm/bionic/memcmp.S | 3 +-- libc/arch-arm/cortex-a9/bionic/memcpy_base.S | 3 +-- libc/arch-arm/cortex-a9/bionic/memset.S | 6 ++---- libc/arch-arm/generic/bionic/memcpy.S | 6 ++---- libc/arch-arm/generic/bionic/memset.S | 6 ++---- 5 files changed, 8 insertions(+), 16 deletions(-) diff --git a/libc/arch-arm/bionic/memcmp.S b/libc/arch-arm/bionic/memcmp.S index c78dbd4bf..6643d5515 100644 --- a/libc/arch-arm/bionic/memcmp.S +++ b/libc/arch-arm/bionic/memcmp.S @@ -221,8 +221,7 @@ ENTRY(memcmp) bne 8b 9: /* restore registers and return */ - ldmfd sp!, {r4, lr} - bx lr + ldmfd sp!, {r4, pc} 10: /* process less than 12 bytes */ cmp r2, #0 diff --git a/libc/arch-arm/cortex-a9/bionic/memcpy_base.S b/libc/arch-arm/cortex-a9/bionic/memcpy_base.S index 5e813050a..6ab5a694b 100644 --- a/libc/arch-arm/cortex-a9/bionic/memcpy_base.S +++ b/libc/arch-arm/cortex-a9/bionic/memcpy_base.S @@ -133,8 +133,7 @@ ENTRY_PRIVATE(MEMCPY_BASE) strbcs ip, [r0], #1 strbcs lr, [r0], #1 - ldmfd sp!, {r0, lr} - bx lr + ldmfd sp!, {r0, pc} END(MEMCPY_BASE) ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED) diff --git a/libc/arch-arm/cortex-a9/bionic/memset.S b/libc/arch-arm/cortex-a9/bionic/memset.S index 8ee6ac26c..48ba81521 100644 --- a/libc/arch-arm/cortex-a9/bionic/memset.S +++ b/libc/arch-arm/cortex-a9/bionic/memset.S @@ -136,8 +136,7 @@ ENTRY_PRIVATE(__memset_large_copy) strbcs r1, [r0], #1 strbmi r1, [r0], #1 subs r2, r2, r3 - popls {r0, r4-r7, lr} /* return */ - bxls lr + popls {r0, r4-r7, pc} /* return */ /* align the destination to a cache-line */ mov r12, r1 @@ -180,8 +179,7 @@ ENTRY_PRIVATE(__memset_large_copy) strhmi r1, [r0], #2 movs r2, r2, lsl #2 strbcs r1, [r0] - ldmfd sp!, {r0, r4-r7, lr} - bx lr + ldmfd sp!, {r0, r4-r7, pc} END(__memset_large_copy) .data diff --git a/libc/arch-arm/generic/bionic/memcpy.S b/libc/arch-arm/generic/bionic/memcpy.S index ea5a3998b..65cba4c5e 100644 --- a/libc/arch-arm/generic/bionic/memcpy.S +++ b/libc/arch-arm/generic/bionic/memcpy.S @@ -194,8 +194,7 @@ ENTRY(memcpy) /* we're done! restore everything and return */ 1: ldmfd sp!, {r5-r11} - ldmfd sp!, {r0, r4, lr} - bx lr + ldmfd sp!, {r0, r4, pc} /********************************************************************/ @@ -385,8 +384,7 @@ ENTRY(memcpy) /* we're done! restore sp and spilled registers and return */ add sp, sp, #28 - ldmfd sp!, {r0, r4, lr} - bx lr + ldmfd sp!, {r0, r4, pc} END(memcpy) // Only reached when the __memcpy_chk check fails. diff --git a/libc/arch-arm/generic/bionic/memset.S b/libc/arch-arm/generic/bionic/memset.S index d17a9c4e0..b8eabbf83 100644 --- a/libc/arch-arm/generic/bionic/memset.S +++ b/libc/arch-arm/generic/bionic/memset.S @@ -82,8 +82,7 @@ ENTRY(memset) strbcs r1, [r0], #1 strbmi r1, [r0], #1 subs r2, r2, r3 - popls {r0, r4-r7, lr} /* return */ - bxls lr + popls {r0, r4-r7, pc} /* return */ /* align the destination to a cache-line */ mov r12, r1 @@ -126,8 +125,7 @@ ENTRY(memset) strhmi r1, [r0], #2 movs r2, r2, lsl #2 strbcs r1, [r0] - ldmfd sp!, {r0, r4-r7, lr} - bx lr + ldmfd sp!, {r0, r4-r7, pc} END(memset) .data From 3225b83f65e1dcec257f2f9ef55587c32a9eb233 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Mon, 27 Jul 2015 13:51:31 -0700 Subject: [PATCH 87/89] Remove pushes from memsets (krait/cortex-a9). On the path that only uses r0 in both the krait and cortex-a9 memset, remove the push and use r3 instead. In addition, for cortex-a9, remove the artificial function since it's not needed since dwarf unwinding is now supported on arm. Change-Id: Icd39cfb6b8350f44368e022063cd97a6b60d46da --- libc/arch-arm/cortex-a9/bionic/memset.S | 27 ++++++++++--------------- libc/arch-arm/krait/bionic/memset.S | 20 ++++++++---------- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/libc/arch-arm/cortex-a9/bionic/memset.S b/libc/arch-arm/cortex-a9/bionic/memset.S index 48ba81521..b39fcc4ae 100644 --- a/libc/arch-arm/cortex-a9/bionic/memset.S +++ b/libc/arch-arm/cortex-a9/bionic/memset.S @@ -69,12 +69,9 @@ END(bzero) ENTRY(memset) // The neon memset only wins for less than 132. cmp r2, #132 - bhi __memset_large_copy - - stmfd sp!, {r0} - .cfi_def_cfa_offset 4 - .cfi_rel_offset r0, 0 + bhi .L_memset_large_copy + mov r3, r0 vdup.8 q0, r1 /* make sure we have at least 32 bytes to write */ @@ -84,7 +81,7 @@ ENTRY(memset) 1: /* The main loop writes 32 bytes at a time */ subs r2, r2, #32 - vst1.8 {d0 - d3}, [r0]! + vst1.8 {d0 - d3}, [r3]! bhs 1b 2: /* less than 32 left */ @@ -93,22 +90,20 @@ ENTRY(memset) beq 3f // writes 16 bytes, 128-bits aligned - vst1.8 {d0, d1}, [r0]! + vst1.8 {d0, d1}, [r3]! 3: /* write up to 15-bytes (count in r2) */ movs ip, r2, lsl #29 bcc 1f - vst1.8 {d0}, [r0]! + vst1.8 {d0}, [r3]! 1: bge 2f - vst1.32 {d0[0]}, [r0]! + vst1.32 {d0[0]}, [r3]! 2: movs ip, r2, lsl #31 - strbmi r1, [r0], #1 - strbcs r1, [r0], #1 - strbcs r1, [r0], #1 - ldmfd sp!, {r0} + strbmi r1, [r3], #1 + strbcs r1, [r3], #1 + strbcs r1, [r3], #1 bx lr -END(memset) -ENTRY_PRIVATE(__memset_large_copy) +.L_memset_large_copy: /* compute the offset to align the destination * offset = (4-(src&3))&3 = -src & 3 */ @@ -180,7 +175,7 @@ ENTRY_PRIVATE(__memset_large_copy) movs r2, r2, lsl #2 strbcs r1, [r0] ldmfd sp!, {r0, r4-r7, pc} -END(__memset_large_copy) +END(memset) .data error_string: diff --git a/libc/arch-arm/krait/bionic/memset.S b/libc/arch-arm/krait/bionic/memset.S index a4fbe178f..ae05965f4 100644 --- a/libc/arch-arm/krait/bionic/memset.S +++ b/libc/arch-arm/krait/bionic/memset.S @@ -69,10 +69,7 @@ END(bzero) /* memset() returns its first argument. */ ENTRY(memset) - stmfd sp!, {r0} - .cfi_def_cfa_offset 4 - .cfi_rel_offset r0, 0 - + mov r3, r0 vdup.8 q0, r1 /* make sure we have at least 32 bytes to write */ @@ -82,7 +79,7 @@ ENTRY(memset) 1: /* The main loop writes 32 bytes at a time */ subs r2, r2, #32 - vst1.8 {d0 - d3}, [r0]! + vst1.8 {d0 - d3}, [r3]! bhs 1b 2: /* less than 32 left */ @@ -91,18 +88,17 @@ ENTRY(memset) beq 3f // writes 16 bytes, 128-bits aligned - vst1.8 {d0, d1}, [r0]! + vst1.8 {d0, d1}, [r3]! 3: /* write up to 15-bytes (count in r2) */ movs ip, r2, lsl #29 bcc 1f - vst1.8 {d0}, [r0]! + vst1.8 {d0}, [r3]! 1: bge 2f - vst1.32 {d0[0]}, [r0]! + vst1.32 {d0[0]}, [r3]! 2: movs ip, r2, lsl #31 - strbmi r1, [r0], #1 - strbcs r1, [r0], #1 - strbcs r1, [r0], #1 - ldmfd sp!, {r0} + strbmi r1, [r3], #1 + strbcs r1, [r3], #1 + strbcs r1, [r3], #1 bx lr END(memset) From db3fa6f17f3c2af240c7c6f0a7535e7e9265eaa8 Mon Sep 17 00:00:00 2001 From: Men_in_black007 Date: Sun, 6 Sep 2015 08:49:19 -0400 Subject: [PATCH 88/89] cortex-a53: Use cortex-a7/bionic/memset.S instead of cortex-a15/bionic/memset.S https://github.com/android/platform_bionic/blob/master/libc/arch-arm/cortex-a53/cortex-a53.mk#L7 Change-Id: If480227ff06470ef47a9a7294d856689c8c08002 Signed-off-by: Men_in_black007 --- libc/arch-arm/cortex-a53/cortex-a53.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/arch-arm/cortex-a53/cortex-a53.mk b/libc/arch-arm/cortex-a53/cortex-a53.mk index f61263154..3f75ebbfc 100644 --- a/libc/arch-arm/cortex-a53/cortex-a53.mk +++ b/libc/arch-arm/cortex-a53/cortex-a53.mk @@ -4,7 +4,7 @@ libc_bionic_src_files_arm += \ arch-arm/cortex-a53/bionic/__strcpy_chk.S \ libc_bionic_src_files_arm += \ - arch-arm/cortex-a15/bionic/memset.S \ + arch-arm/cortex-a7/bionic/memset.S \ libc_bionic_src_files_arm += \ arch-arm/cortex-a15/bionic/strcat.S \ From 12eb9c556ad50585bf0067974c4db41ce2fe0784 Mon Sep 17 00:00:00 2001 From: "Christopher R. Palmer" Date: Wed, 9 Sep 2015 21:49:32 -0400 Subject: [PATCH 89/89] bionic: Allow devices to add device specfic static libs This provides a convenient (albeit powerful enough to be dangerous) hook to add symbols needed to support vendor blobs that do not necessarily match our source code. Doing so via this hook has several advantages over patching the code in question: * The hacks do no pollute other repositories * The hacks do not have any risk of breaking any other devices * The hacks don't just live forever when we forget they exist * The hacks are all easy to find by locating them together When using this, please take extra care to include only the minimal code to support the change. Keep in mind that all code (and all libraries your code links against) will be part of the address space of every single process in the system! Change-Id: I6dcd9ad7cee330febe1a974619144d378b67b364 --- libc/Android.mk | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libc/Android.mk b/libc/Android.mk index f9bb5b416..aa92ffd19 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -1072,6 +1072,9 @@ LOCAL_SRC_FILES_arm += \ arch-arm/bionic/atexit_legacy.c \ arch-common/bionic/crtend_so.S +# Allow devices to provide additional symbols +LOCAL_WHOLE_STATIC_LIBRARIES += $(BOARD_PROVIDES_ADDITIONAL_BIONIC_STATIC_LIBS) + include $(BUILD_SHARED_LIBRARY)