File: //proc/self/root/usr/include/hphp/util/word-mem.h
/*
+----------------------------------------------------------------------+
| HipHop for PHP |
+----------------------------------------------------------------------+
| Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com) |
+----------------------------------------------------------------------+
| This source file is subject to version 3.01 of the PHP license, |
| that is bundled with this package in the file LICENSE, and is |
| available through the world-wide-web at the following url: |
| http://www.php.net/license/3_01.txt |
| If you did not receive a copy of the PHP license and are unable to |
| obtain it through the world-wide-web, please send a note to |
| license@php.net so we can mail you a copy immediately. |
+----------------------------------------------------------------------+
*/
#ifndef incl_HPHP_WORD_MEM_H_
#define incl_HPHP_WORD_MEM_H_
#include <limits>
#include <folly/Portability.h>
#include "hphp/util/assertions.h"
extern "C" void* _memcpy8(void* dst, const void* src, size_t len);
extern "C" void* _memcpy16(void* dst, const void* src, size_t len);
extern "C" void _bcopy32(void* dst, const void* src, size_t len);
extern "C" void _bcopy_in_64(void* dst, const void* src, size_t lenIn64);
namespace HPHP {
/*
* Specialized memcpy implementations that takes advantage of the known
* properties in length and alignment.
*
* o memcpy8(dst, src, len) is equivalent to
* static_cast<char*>(memcpy(dst, src, (len + 7) / 8 * 8)) + len;
* it returns a char* pointing to dst[len] instead of dst, in order to
* ease its use in string operations.
*
* Note that it could overrun the buffer by up to 7 bytes, depending on len
* and alignment of the buffers. When both src and dst are aligned to 8
* bytes, it is safe. It can also be used in other situations given
* sufficient readable space after the buffers.
*
* o memcpy16(dst, src, len) is equivalent to
* assert(len > 0 && len % 16 == 0);
* memcpy(dst, src, len);
*
* o bcopy32(dst, src, len) is equivalent to
* assert(len >= 32);
* memcpy(dst, src, len / 32 * 32);
* except that it returns void.
*
* o bcopy_in_64(dst, src, lenIn64) is equivalent to
* assert(lenIn64 > 0);
* memcpy(dst, src, 64 * lenIn64);
* except that it returns void.
*/
inline char* memcpy8(void* dst, const void* src, uint32_t len) {
#if defined(__x86_64__)
return reinterpret_cast<char*>(_memcpy8(dst, src, len));
#else
memcpy(dst, src, len);
return reinterpret_cast<char*>(dst) + len;
#endif
}
inline char* memcpy16(void* dst, const void* src, uint32_t len) {
assertx(len > 0 && len % 16 == 0);
#if defined(__x86_64__)
return reinterpret_cast<char*>(_memcpy16(dst, src, len));
#else
return reinterpret_cast<char*>(memcpy(dst, src, len));
#endif
}
inline void bcopy32(void* dst, const void* src, uint32_t len) {
assertx(len >= 32);
#if defined(__x86_64__)
_bcopy32(dst, src, len);
#else
memcpy(dst, src, len / 32 * 32);
#endif
}
inline void bcopy_in_64(void* dst, const void* src, uint32_t lenIn64) {
assertx(lenIn64 != 0);
#if defined(__x86_64__)
_bcopy_in_64(dst, src, lenIn64);
#else
memcpy(dst, src, lenIn64 * 64);
#endif
}
// Inline assembly version to avoid a function call.
inline void bcopy32_inline(void* dst, const void* src, uint32_t len) {
assertx(len >= 32);
#if defined(__x86_64__)
__asm__ __volatile__("shr $5, %0\n"
".LBCP32%=:\n"
"movdqu (%1), %%xmm0\n"
"movdqu 16(%1), %%xmm1\n"
"add $32, %1\n"
"movdqu %%xmm0, (%2)\n"
"movdqu %%xmm1, 16(%2)\n"
"add $32, %2\n"
"dec %0\n"
"jg .LBCP32%=\n"
: "+r"(len), "+r"(src), "+r"(dst)
:: "xmm0", "xmm1"
);
#elif defined(__aarch64__)
int64_t t3, t4, t5, t6, t7;
__asm__ __volatile__("lsr %x0, %x0, #5\n"
"sub %x1, %x1, #16\n"
"sub %x2, %x2, #16\n"
".LBCP32%=:\n"
"ldp %x3, %x4, [%x1, #16]\n"
"ldp %x5, %x6, [%x1, #32]!\n"
"stp %x3, %x4, [%x2, #16]\n"
"stp %x5, %x6, [%x2, #32]!\n"
"subs %x0, %x0, #1\n"
"bgt .LBCP32%=\n"
: "+r"(len), "+r"(src), "+r"(dst),
"=r"(t3), "=r"(t4), "=r"(t5), "=r"(t6), "=r"(t7)
);
#else
bcopy32(dst, src, len);
#endif
}
inline void memcpy16_inline(void* dst, const void* src, uint64_t len) {
assertx(len >=16 && len % 16 == 0);
#if defined(__x86_64__)
__asm__ __volatile__("movdqu -16(%1, %0), %%xmm0\n"
"movdqu %%xmm0, -16(%2, %0)\n"
"shr $5, %0\n"
"jz .LEND%=\n"
".LR32%=:\n"
"movdqu (%1), %%xmm0\n"
"movdqu 16(%1), %%xmm1\n"
"add $32, %1\n"
"movdqu %%xmm0, (%2)\n"
"movdqu %%xmm1, 16(%2)\n"
"add $32, %2\n"
"dec %0\n"
"jg .LR32%=\n"
".LEND%=:\n"
: "+r"(len), "+r"(src), "+r"(dst)
:: "xmm0", "xmm1"
);
#elif defined(__aarch64__)
int64_t t3, t4, t5, t6, s1, d1, d2;
__asm__ __volatile__("mov %x7, %x1\n"
"add %x1, %x1, %x0\n"
"ldp %x3, %x4, [%x1, #-16]!\n"
"mov %x8, %x2\n"
"add %x2, %x2, %x0\n"
"stp %x3, %x4, [%x2, #-16]!\n"
"lsr %x0, %x0, #5\n"
"cbz %x0, .LEND%=\n"
"sub %x7, %x7, #16\n"
"sub %x8, %x8, #16\n"
".LR32%=:\n"
"ldp %x3, %x4, [%x7, #16]\n"
"ldp %x5, %x6, [%x7, #32]!\n"
"stp %x3, %x4, [%x8, #16]\n"
"stp %x5, %x6, [%x8, #32]!\n"
"subs %x0, %x0, #1\n"
"bgt .LR32%=\n"
".LEND%=:\n"
: "+r"(len), "+r"(src), "+r"(dst),
"=r"(t3), "=r"(t4), "=r"(t5), "=r"(t6),
"=r"(s1), "=r"(d1), "=r"(d2)
);
#else
memcpy16(dst, src, len);
#endif
}
//////////////////////////////////////////////////////////////////////
// ASan is less precise than valgrind and believes this function overruns reads
#ifndef FOLLY_SANITIZE_ADDRESS
/*
* Word at a time comparison for two strings of length `lenBytes'. Returns
* true if the regions are the same. This should be invoked only when we know
* the two strings have the same length. It will not check for the null
* terminator.
*
* Assumes it can load more words than the size to compare (this is often
* possible in HPHP when you know you dealing with request-allocated memory).
* The final word compare is adjusted to handle the slack in lenBytes so only
* the bytes we care about are compared.
*
* Assumes that the the buffer addresses are 8-bytes aligned.
*/
ALWAYS_INLINE
bool wordsame(const void* mem1, const void* mem2, uint32_t lenBytes) {
using T = uint64_t;
auto constexpr W = sizeof(T);
assert(reinterpret_cast<const uintptr_t>(mem1) % W == 0);
assert(reinterpret_cast<const uintptr_t>(mem2) % W == 0);
// Inverse of lenBytes. Do the negation here to avoid doing it later on the
// critical path.
int32_t const nBytes = -lenBytes;
// Check if `lenBytes' is 0, do the check right here to reuse the flags of
// the neg instruction. This saves a test instruction.
if (UNLIKELY(nBytes == 0)) return true;
// Do the shift here to avoid doing it later on the critical path. But we
// will have to switch to 64 bit here to support very long strings.
int64_t nBits = static_cast<int64_t>(nBytes) * 8u;
// Use the base+index addressing mode in x86, so that we only need to
// increment the base pointer in the loop.
auto p1 = reinterpret_cast<intptr_t>(mem1);
auto const diff = reinterpret_cast<intptr_t>(mem2) - p1;
T data;
do {
data = *(reinterpret_cast<const T*>(p1));
data ^= *(reinterpret_cast<const T*>(p1 + diff));
p1 += W;
nBits += W * 8;
if (nBits >= 0) {
// As a note for future consideration, we could consider precomputing a
// 64-bit mask, so that the fraction of the last qword can be checked
// faster. But that would require an additional register for the
// mask. So it depends on register pressure of the call site.
return !(data << nBits);
}
} while (data == 0);
return false;
}
#else // FOLLY_SANITIZE_ADDRESS
ALWAYS_INLINE
bool wordsame(const void* mem1, const void* mem2, size_t lenBytes) {
assert(reinterpret_cast<const uintptr_t>(mem1) % 4 == 0);
return !memcmp(mem1, mem2, lenBytes);
}
#endif
/*
* Like memcpy, but copies numT POD values 8 bytes at a time.
* The actual number of bytes copied must be a multiple of 8.
*/
template<class T>
T* wordcpy(T* to, const T* from, size_t numT) {
assert(numT < std::numeric_limits<int64_t>::max() &&
(numT * sizeof(T)) % 8 == 0);
size_t numWords = numT * sizeof(T) / 8;
assert(numWords != 0);
auto d = (int64_t*)to;
auto s = (int64_t*)from;
do {
*d++ = *s++;
} while (--numWords);
return to;
}
/*
* Fills a memory area with ones, 8 bytes at a time.
*/
template<class T>
T* wordfillones(T* ptr, size_t numT) {
assert(numT < std::numeric_limits<int64_t>::max() &&
(numT * sizeof(T)) % 8 == 0);
assert(numT != 0);
auto numWords = numT * sizeof(T) / 8;
auto d = (int64_t*)ptr;
do {
*d++ = -1;
} while (--numWords);
return ptr;
}
//////////////////////////////////////////////////////////////////////
}
#endif