First Commit

externals/boost/libs/atomic/config/Jamfile.v2

@@ -0,0 +1,20 @@
+#
+#             Copyright Andrey Semashev 2020.
+# Distributed under the Boost Software License, Version 1.0.
+#    (See accompanying file LICENSE_1_0.txt or copy at
+#          http://www.boost.org/LICENSE_1_0.txt)
+#
+
+import atomic-arch-config ;
+
+lib synchronization ;
+explicit synchronization ;
+
+exe has_synchronization_lib : has_synchronization_lib.cpp : <library>synchronization ;
+explicit has_synchronization_lib ;
+
+obj has_sse2 : has_sse2.cpp : <conditional>@atomic-arch-config.sse2-flags ;
+explicit has_sse2 ;
+
+obj has_sse41 : has_sse41.cpp : <conditional>@atomic-arch-config.sse41-flags ;
+explicit has_sse41 ;

externals/boost/libs/atomic/config/has_sse2.cpp

@@ -0,0 +1,17 @@
+/*
+ *             Copyright Andrey Semashev 2020.
+ * Distributed under the Boost Software License, Version 1.0.
+ *    (See accompanying file LICENSE_1_0.txt or copy at
+ *          http://www.boost.org/LICENSE_1_0.txt)
+ */
+
+#include <emmintrin.h>
+
+int main(int, char*[])
+{
+    __m128i mm = _mm_setzero_si128();
+    mm = _mm_cmpeq_epi32(mm, mm);
+    mm = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm), _mm_castsi128_ps(mm), _MM_SHUFFLE(2, 0, 2, 0)));
+    mm = _mm_packs_epi32(mm, mm);
+    return _mm_movemask_epi8(mm);
+}

externals/boost/libs/atomic/config/has_sse41.cpp

@@ -0,0 +1,17 @@
+/*
+ *             Copyright Andrey Semashev 2020.
+ * Distributed under the Boost Software License, Version 1.0.
+ *    (See accompanying file LICENSE_1_0.txt or copy at
+ *          http://www.boost.org/LICENSE_1_0.txt)
+ */
+
+#include <smmintrin.h>
+
+int main(int, char*[])
+{
+    __m128i mm = _mm_setzero_si128();
+    mm = _mm_cmpeq_epi64(mm, mm);
+    mm = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm), _mm_castsi128_ps(mm), _MM_SHUFFLE(2, 0, 2, 0)));
+    mm = _mm_packs_epi32(mm, mm);
+    return _mm_movemask_epi8(mm);
+}

externals/boost/libs/atomic/config/has_synchronization_lib.cpp

@@ -0,0 +1,33 @@
+/*
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * Copyright (c) 2020 Andrey Semashev
+ */
+
+#define BOOST_USE_WINAPI_VERSION 0x0602
+
+// Include Boost.Predef first so that windows.h is guaranteed to be not included
+#include <boost/predef/os/windows.h>
+#include <boost/predef/os/cygwin.h>
+#if !BOOST_OS_WINDOWS && !BOOST_OS_CYGWIN
+#error "This config test is for Windows only"
+#endif
+
+#include <boost/winapi/config.hpp>
+#include <boost/predef/platform.h>
+#if !(BOOST_USE_WINAPI_VERSION >= BOOST_WINAPI_VERSION_WIN8 && (BOOST_WINAPI_PARTITION_APP || BOOST_WINAPI_PARTITION_SYSTEM))
+#error "No WaitOnAddress API"
+#endif
+
+#include <cstddef>
+#include <boost/winapi/basic_types.hpp>
+#include <boost/winapi/wait_on_address.hpp>
+
+int main()
+{
+    unsigned int n = 0u, compare = 0u;
+    boost::winapi::WaitOnAddress(&n, &compare, sizeof(n), 0);
+    return 0;
+}

externals/boost/libs/atomic/src/bit_operation_tools.hpp

@@ -0,0 +1,82 @@
+/*
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * Copyright (c) 2020 Andrey Semashev
+ */
+/*!
+ * \file   bit_operation_tools.hpp
+ *
+ * This file contains bit operation tools
+ */
+
+#ifndef BOOST_ATOMIC_BIT_OPERATION_TOOLS_HPP_INCLUDED_
+#define BOOST_ATOMIC_BIT_OPERATION_TOOLS_HPP_INCLUDED_
+
+#include <boost/predef/architecture/x86.h>
+
+#if BOOST_ARCH_X86
+
+#include <boost/atomic/detail/config.hpp>
+#include <boost/atomic/detail/header.hpp>
+
+#if defined(_MSC_VER)
+extern "C" unsigned char _BitScanForward(unsigned long* index, unsigned long x);
+#if defined(BOOST_MSVC)
+#pragma intrinsic(_BitScanForward)
+#endif
+#endif
+
+namespace boost {
+namespace atomics {
+namespace detail {
+
+//! Counts trailing zero bits
+BOOST_FORCEINLINE unsigned int count_trailing_zeros(unsigned int x)
+{
+#if defined(__GNUC__)
+    return __builtin_ctz(x);
+#elif defined(_MSC_VER)
+    unsigned long index;
+    _BitScanForward(&index, x);
+    return static_cast< unsigned int >(index);
+#else
+    unsigned int index = 0u;
+    if ((x & 0xFFFF) == 0u)
+    {
+        x >>= 16;
+        index += 16u;
+    }
+    if ((x & 0xFF) == 0u)
+    {
+        x >>= 8;
+        index += 8u;
+    }
+    if ((x & 0xF) == 0u)
+    {
+        x >>= 4;
+        index += 4u;
+    }
+    if ((x & 0x3) == 0u)
+    {
+        x >>= 2;
+        index += 2u;
+    }
+    if ((x & 0x1) == 0u)
+    {
+        index += 1u;
+    }
+    return index;
+#endif
+}
+
+} // namespace detail
+} // namespace atomics
+} // namespace boost
+
+#include <boost/atomic/detail/footer.hpp>
+
+#endif // BOOST_ARCH_X86
+
+#endif // BOOST_ATOMIC_BIT_OPERATION_TOOLS_HPP_INCLUDED_

externals/boost/libs/atomic/src/cpuid.hpp

@@ -0,0 +1,86 @@
+/*
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * Copyright (c) 2020 Andrey Semashev
+ */
+/*!
+ * \file   cpuid.hpp
+ *
+ * This file contains declaration of \c cpuid function
+ */
+
+#ifndef BOOST_ATOMIC_CPUID_HPP_INCLUDED_
+#define BOOST_ATOMIC_CPUID_HPP_INCLUDED_
+
+#include <boost/predef/architecture/x86.h>
+
+#if BOOST_ARCH_X86
+
+#if defined(_MSC_VER)
+#include <intrin.h> // __cpuid
+#endif
+#include <boost/cstdint.hpp>
+#include <boost/atomic/detail/config.hpp>
+
+#include <boost/atomic/detail/header.hpp>
+
+namespace boost {
+namespace atomics {
+namespace detail {
+
+//! The function invokes x86 cpuid instruction
+inline void cpuid(uint32_t& eax, uint32_t& ebx, uint32_t& ecx, uint32_t& edx)
+{
+#if defined(__GNUC__)
+#if (defined(__i386__) || defined(__VXWORKS__)) && (defined(__PIC__) || defined(__PIE__)) && !(defined(__clang__) || (defined(BOOST_GCC) && BOOST_GCC >= 50100))
+    // Unless the compiler can do it automatically, we have to backup ebx in 32-bit PIC/PIE code because it is reserved by the ABI.
+    // For VxWorks ebx is reserved on 64-bit as well.
+#if defined(__x86_64__)
+    uint64_t rbx = ebx;
+    __asm__ __volatile__
+    (
+        "xchgq %%rbx, %0\n\t"
+        "cpuid\n\t"
+        "xchgq %%rbx, %0\n\t"
+            : "+DS" (rbx), "+a" (eax), "+c" (ecx), "+d" (edx)
+    );
+    ebx = static_cast< uint32_t >(rbx);
+#else // defined(__x86_64__)
+    __asm__ __volatile__
+    (
+        "xchgl %%ebx, %0\n\t"
+        "cpuid\n\t"
+        "xchgl %%ebx, %0\n\t"
+            : "+DS" (ebx), "+a" (eax), "+c" (ecx), "+d" (edx)
+    );
+#endif // defined(__x86_64__)
+#else
+    __asm__ __volatile__
+    (
+        "cpuid\n\t"
+            : "+a" (eax), "+b" (ebx), "+c" (ecx), "+d" (edx)
+    );
+#endif
+#elif defined(_MSC_VER)
+    int regs[4] = {};
+    __cpuid(regs, eax);
+    eax = regs[0];
+    ebx = regs[1];
+    ecx = regs[2];
+    edx = regs[3];
+#else
+#error "Boost.Atomic: Unsupported compiler, cpuid instruction cannot be generated"
+#endif
+}
+
+} // namespace detail
+} // namespace atomics
+} // namespace boost
+
+#include <boost/atomic/detail/footer.hpp>
+
+#endif // BOOST_ARCH_X86
+
+#endif // BOOST_ATOMIC_CPUID_HPP_INCLUDED_

externals/boost/libs/atomic/src/find_address.hpp

@@ -0,0 +1,45 @@
+/*
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * Copyright (c) 2020 Andrey Semashev
+ */
+/*!
+ * \file   find_address.hpp
+ *
+ * This file contains declaration of \c find_address algorithm
+ */
+
+#ifndef BOOST_ATOMIC_FIND_ADDRESS_HPP_INCLUDED_
+#define BOOST_ATOMIC_FIND_ADDRESS_HPP_INCLUDED_
+
+#include <cstddef>
+#include <boost/predef/architecture/x86.h>
+#include <boost/atomic/detail/config.hpp>
+#include <boost/atomic/detail/int_sizes.hpp>
+#include <boost/atomic/detail/header.hpp>
+
+namespace boost {
+namespace atomics {
+namespace detail {
+
+//! \c find_address signature
+typedef std::size_t (find_address_t)(const volatile void* addr, const volatile void* const* addrs, std::size_t size);
+
+extern find_address_t find_address_generic;
+
+#if BOOST_ARCH_X86 && defined(BOOST_ATOMIC_DETAIL_SIZEOF_POINTER) && (BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8 || BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 4)
+extern find_address_t find_address_sse2;
+#if BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8
+extern find_address_t find_address_sse41;
+#endif // BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8
+#endif // BOOST_ARCH_X86 && defined(BOOST_ATOMIC_DETAIL_SIZEOF_POINTER) && (BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8 || BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 4)
+
+} // namespace detail
+} // namespace atomics
+} // namespace boost
+
+#include <boost/atomic/detail/footer.hpp>
+
+#endif // BOOST_ATOMIC_FIND_ADDRESS_HPP_INCLUDED_

externals/boost/libs/atomic/src/find_address_sse2.cpp

@@ -0,0 +1,289 @@
+/*
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * Copyright (c) 2020 Andrey Semashev
+ */
+/*!
+ * \file   find_address_sse2.cpp
+ *
+ * This file contains SSE2 implementation of the \c find_address algorithm
+ */
+
+#include <boost/predef/architecture/x86.h>
+#include <boost/atomic/detail/int_sizes.hpp>
+
+#if BOOST_ARCH_X86 && defined(BOOST_ATOMIC_DETAIL_SIZEOF_POINTER) && (BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8 || BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 4)
+
+#include <cstddef>
+#include <emmintrin.h>
+
+#include <boost/cstdint.hpp>
+#include <boost/atomic/detail/config.hpp>
+#include <boost/atomic/detail/intptr.hpp>
+#include "find_address.hpp"
+#include "x86_vector_tools.hpp"
+#include "bit_operation_tools.hpp"
+
+#include <boost/atomic/detail/header.hpp>
+
+namespace boost {
+namespace atomics {
+namespace detail {
+
+#if BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8
+namespace {
+
+BOOST_FORCEINLINE __m128i mm_pand_si128(__m128i mm1, __m128i mm2)
+{
+    // As of 2020, gcc, clang and icc prefer to generate andps instead of pand if the surrounding
+    // instructions pertain to FP domain, even if we use the _mm_and_si128 intrinsic. In our
+    // algorithm implementation, the FP instruction happen to be shufps, which is not actually
+    // restricted to FP domain (it is actually implemented in a separate MMX EU in Pentium 4 or
+    // a shuffle EU in INT domain in Core 2; on AMD K8/K10 all SSE instructions are implemented in
+    // FADD, FMUL and FMISC EUs regardless of INT/FP data types, and shufps is implemented in FADD/FMUL).
+    // In other words, there should be no domain bypass penalty between shufps and pand.
+    //
+    // This would usually not pose a problem since andps and pand have the same latency and throughput
+    // on most architectures of that age (before SSE4.1). However, it is possible that a newer architecture
+    // runs the SSE2 code path (e.g. because some weird compiler doesn't support SSE4.1 or because
+    // a hypervisor blocks SSE4.1 detection), and there pand may have a better throughput. For example,
+    // Sandy Bridge can execute 3 pand instructions per cycle, but only one andps. For this reason
+    // we prefer to generate pand and not andps.
+#if defined(__GNUC__)
+#if defined(__AVX__)
+    // Generate VEX-coded variant if the code is compiled for AVX and later.
+    __asm__("vpand %1, %0, %0\n\t" : "+x" (mm1) : "x" (mm2));
+#else
+    __asm__("pand %1, %0\n\t" : "+x" (mm1) : "x" (mm2));
+#endif
+#else
+    mm1 = _mm_and_si128(mm1, mm2);
+#endif
+    return mm1;
+}
+
+} // namespace
+#endif // BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8
+
+//! SSE2 implementation of the \c find_address algorithm
+std::size_t find_address_sse2(const volatile void* addr, const volatile void* const* addrs, std::size_t size)
+{
+#if BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8
+
+    if (size < 12u)
+        return find_address_generic(addr, addrs, size);
+
+    const __m128i mm_addr = mm_set1_epiptr((uintptr_t)addr);
+    std::size_t pos = 0u;
+    const std::size_t n = (size + 1u) & ~static_cast< std::size_t >(1u);
+    for (std::size_t m = n & ~static_cast< std::size_t >(15u); pos < m; pos += 16u)
+    {
+        __m128i mm1 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos));
+        __m128i mm2 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 2u));
+        __m128i mm3 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 4u));
+        __m128i mm4 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 6u));
+        __m128i mm5 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 8u));
+        __m128i mm6 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 10u));
+        __m128i mm7 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 12u));
+        __m128i mm8 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 14u));
+
+        mm1 = _mm_cmpeq_epi32(mm1, mm_addr);
+        mm2 = _mm_cmpeq_epi32(mm2, mm_addr);
+        mm3 = _mm_cmpeq_epi32(mm3, mm_addr);
+        mm4 = _mm_cmpeq_epi32(mm4, mm_addr);
+        mm5 = _mm_cmpeq_epi32(mm5, mm_addr);
+        mm6 = _mm_cmpeq_epi32(mm6, mm_addr);
+        mm7 = _mm_cmpeq_epi32(mm7, mm_addr);
+        mm8 = _mm_cmpeq_epi32(mm8, mm_addr);
+
+        __m128i mm_mask1_lo = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm1), _mm_castsi128_ps(mm2), _MM_SHUFFLE(2, 0, 2, 0)));
+        __m128i mm_mask1_hi = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm1), _mm_castsi128_ps(mm2), _MM_SHUFFLE(3, 1, 3, 1)));
+
+        __m128i mm_mask2_lo = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm3), _mm_castsi128_ps(mm4), _MM_SHUFFLE(2, 0, 2, 0)));
+        __m128i mm_mask2_hi = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm3), _mm_castsi128_ps(mm4), _MM_SHUFFLE(3, 1, 3, 1)));
+
+        __m128i mm_mask3_lo = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm5), _mm_castsi128_ps(mm6), _MM_SHUFFLE(2, 0, 2, 0)));
+        __m128i mm_mask3_hi = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm5), _mm_castsi128_ps(mm6), _MM_SHUFFLE(3, 1, 3, 1)));
+
+        __m128i mm_mask4_lo = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm7), _mm_castsi128_ps(mm8), _MM_SHUFFLE(2, 0, 2, 0)));
+        __m128i mm_mask4_hi = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm7), _mm_castsi128_ps(mm8), _MM_SHUFFLE(3, 1, 3, 1)));
+
+        mm_mask1_lo = mm_pand_si128(mm_mask1_lo, mm_mask1_hi);
+        mm_mask2_lo = mm_pand_si128(mm_mask2_lo, mm_mask2_hi);
+        mm_mask3_lo = mm_pand_si128(mm_mask3_lo, mm_mask3_hi);
+        mm_mask4_lo = mm_pand_si128(mm_mask4_lo, mm_mask4_hi);
+
+        mm_mask1_lo = _mm_packs_epi32(mm_mask1_lo, mm_mask2_lo);
+        mm_mask3_lo = _mm_packs_epi32(mm_mask3_lo, mm_mask4_lo);
+
+        mm_mask1_lo = _mm_packs_epi16(mm_mask1_lo, mm_mask3_lo);
+
+        uint32_t mask = _mm_movemask_epi8(mm_mask1_lo);
+        if (mask)
+        {
+            pos += atomics::detail::count_trailing_zeros(mask);
+            goto done;
+        }
+    }
+
+    if ((n - pos) >= 8u)
+    {
+        __m128i mm1 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos));
+        __m128i mm2 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 2u));
+        __m128i mm3 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 4u));
+        __m128i mm4 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 6u));
+
+        mm1 = _mm_cmpeq_epi32(mm1, mm_addr);
+        mm2 = _mm_cmpeq_epi32(mm2, mm_addr);
+        mm3 = _mm_cmpeq_epi32(mm3, mm_addr);
+        mm4 = _mm_cmpeq_epi32(mm4, mm_addr);
+
+        __m128i mm_mask1_lo = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm1), _mm_castsi128_ps(mm2), _MM_SHUFFLE(2, 0, 2, 0)));
+        __m128i mm_mask1_hi = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm1), _mm_castsi128_ps(mm2), _MM_SHUFFLE(3, 1, 3, 1)));
+
+        __m128i mm_mask2_lo = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm3), _mm_castsi128_ps(mm4), _MM_SHUFFLE(2, 0, 2, 0)));
+        __m128i mm_mask2_hi = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm3), _mm_castsi128_ps(mm4), _MM_SHUFFLE(3, 1, 3, 1)));
+
+        mm_mask1_lo = mm_pand_si128(mm_mask1_lo, mm_mask1_hi);
+        mm_mask2_lo = mm_pand_si128(mm_mask2_lo, mm_mask2_hi);
+
+        mm_mask1_lo = _mm_packs_epi32(mm_mask1_lo, mm_mask2_lo);
+
+        uint32_t mask = _mm_movemask_epi8(mm_mask1_lo);
+        if (mask)
+        {
+            pos += atomics::detail::count_trailing_zeros(mask) / 2u;
+            goto done;
+        }
+
+        pos += 8u;
+    }
+
+    if ((n - pos) >= 4u)
+    {
+        __m128i mm1 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos));
+        __m128i mm2 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 2u));
+
+        mm1 = _mm_cmpeq_epi32(mm1, mm_addr);
+        mm2 = _mm_cmpeq_epi32(mm2, mm_addr);
+
+        __m128i mm_mask1_lo = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm1), _mm_castsi128_ps(mm2), _MM_SHUFFLE(2, 0, 2, 0)));
+        __m128i mm_mask1_hi = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm1), _mm_castsi128_ps(mm2), _MM_SHUFFLE(3, 1, 3, 1)));
+
+        mm_mask1_lo = mm_pand_si128(mm_mask1_lo, mm_mask1_hi);
+
+        uint32_t mask = _mm_movemask_ps(_mm_castsi128_ps(mm_mask1_lo));
+        if (mask)
+        {
+            pos += atomics::detail::count_trailing_zeros(mask);
+            goto done;
+        }
+
+        pos += 4u;
+    }
+
+    if (pos < n)
+    {
+        __m128i mm1 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos));
+
+        mm1 = _mm_cmpeq_epi32(mm1, mm_addr);
+        __m128i mm_mask = _mm_shuffle_epi32(mm1, _MM_SHUFFLE(2, 3, 0, 1));
+        mm_mask = mm_pand_si128(mm_mask, mm1);
+
+        uint32_t mask = _mm_movemask_pd(_mm_castsi128_pd(mm_mask));
+        if (mask)
+        {
+            pos += atomics::detail::count_trailing_zeros(mask);
+            goto done;
+        }
+
+        pos += 2u;
+    }
+
+done:
+    return pos;
+
+#else // BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8
+
+    if (size < 10u)
+        return find_address_generic(addr, addrs, size);
+
+    const __m128i mm_addr = _mm_set1_epi32((uintptr_t)addr);
+    std::size_t pos = 0u;
+    const std::size_t n = (size + 3u) & ~static_cast< std::size_t >(3u);
+    for (std::size_t m = n & ~static_cast< std::size_t >(15u); pos < m; pos += 16u)
+    {
+        __m128i mm1 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos));
+        __m128i mm2 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 4u));
+        __m128i mm3 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 8u));
+        __m128i mm4 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 12u));
+
+        mm1 = _mm_cmpeq_epi32(mm1, mm_addr);
+        mm2 = _mm_cmpeq_epi32(mm2, mm_addr);
+        mm3 = _mm_cmpeq_epi32(mm3, mm_addr);
+        mm4 = _mm_cmpeq_epi32(mm4, mm_addr);
+
+        mm1 = _mm_packs_epi32(mm1, mm2);
+        mm3 = _mm_packs_epi32(mm3, mm4);
+
+        mm1 = _mm_packs_epi16(mm1, mm3);
+
+        uint32_t mask = _mm_movemask_epi8(mm1);
+        if (mask)
+        {
+            pos += atomics::detail::count_trailing_zeros(mask);
+            goto done;
+        }
+    }
+
+    if ((n - pos) >= 8u)
+    {
+        __m128i mm1 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos));
+        __m128i mm2 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 4u));
+
+        mm1 = _mm_cmpeq_epi32(mm1, mm_addr);
+        mm2 = _mm_cmpeq_epi32(mm2, mm_addr);
+
+        mm1 = _mm_packs_epi32(mm1, mm2);
+
+        uint32_t mask = _mm_movemask_epi8(mm1);
+        if (mask)
+        {
+            pos += atomics::detail::count_trailing_zeros(mask) / 2u;
+            goto done;
+        }
+
+        pos += 8u;
+    }
+
+    if (pos < n)
+    {
+        __m128i mm1 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos));
+
+        mm1 = _mm_cmpeq_epi32(mm1, mm_addr);
+
+        uint32_t mask = _mm_movemask_ps(_mm_castsi128_ps(mm1));
+        if (mask)
+        {
+            pos += atomics::detail::count_trailing_zeros(mask);
+            goto done;
+        }
+
+        pos += 4u;
+    }
+
+done:
+    return pos;
+
+#endif // BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8
+}
+
+} // namespace detail
+} // namespace atomics
+} // namespace boost
+
+#include <boost/atomic/detail/footer.hpp>
+
+#endif // BOOST_ARCH_X86 && defined(BOOST_ATOMIC_DETAIL_SIZEOF_POINTER) && (BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8 || BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 4)

externals/boost/libs/atomic/src/find_address_sse41.cpp

@@ -0,0 +1,154 @@
+/*
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * Copyright (c) 2020 Andrey Semashev
+ */
+/*!
+ * \file   find_address_sse41.cpp
+ *
+ * This file contains SSE4.1 implementation of the \c find_address algorithm
+ */
+
+#include <boost/predef/architecture/x86.h>
+#include <boost/atomic/detail/int_sizes.hpp>
+
+#if BOOST_ARCH_X86 && defined(BOOST_ATOMIC_DETAIL_SIZEOF_POINTER) && (BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8)
+
+#include <cstddef>
+#include <smmintrin.h>
+
+#include <boost/cstdint.hpp>
+#include <boost/atomic/detail/config.hpp>
+#include <boost/atomic/detail/intptr.hpp>
+#include "find_address.hpp"
+#include "x86_vector_tools.hpp"
+#include "bit_operation_tools.hpp"
+
+#include <boost/atomic/detail/header.hpp>
+
+namespace boost {
+namespace atomics {
+namespace detail {
+
+//! SSE4.1 implementation of the \c find_address algorithm
+std::size_t find_address_sse41(const volatile void* addr, const volatile void* const* addrs, std::size_t size)
+{
+    if (size < 12u)
+        return find_address_generic(addr, addrs, size);
+
+    const __m128i mm_addr = mm_set1_epiptr((uintptr_t)addr);
+    std::size_t pos = 0u;
+    const std::size_t n = (size + 1u) & ~static_cast< std::size_t >(1u);
+    for (std::size_t m = n & ~static_cast< std::size_t >(15u); pos < m; pos += 16u)
+    {
+        __m128i mm1 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos));
+        __m128i mm2 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 2u));
+        __m128i mm3 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 4u));
+        __m128i mm4 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 6u));
+        __m128i mm5 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 8u));
+        __m128i mm6 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 10u));
+        __m128i mm7 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 12u));
+        __m128i mm8 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 14u));
+
+        mm1 = _mm_cmpeq_epi64(mm1, mm_addr);
+        mm2 = _mm_cmpeq_epi64(mm2, mm_addr);
+        mm3 = _mm_cmpeq_epi64(mm3, mm_addr);
+        mm4 = _mm_cmpeq_epi64(mm4, mm_addr);
+        mm5 = _mm_cmpeq_epi64(mm5, mm_addr);
+        mm6 = _mm_cmpeq_epi64(mm6, mm_addr);
+        mm7 = _mm_cmpeq_epi64(mm7, mm_addr);
+        mm8 = _mm_cmpeq_epi64(mm8, mm_addr);
+
+        mm1 = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm1), _mm_castsi128_ps(mm2), _MM_SHUFFLE(2, 0, 2, 0)));
+        mm3 = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm3), _mm_castsi128_ps(mm4), _MM_SHUFFLE(2, 0, 2, 0)));
+        mm5 = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm5), _mm_castsi128_ps(mm6), _MM_SHUFFLE(2, 0, 2, 0)));
+        mm7 = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm7), _mm_castsi128_ps(mm8), _MM_SHUFFLE(2, 0, 2, 0)));
+
+        mm1 = _mm_packs_epi32(mm1, mm3);
+        mm5 = _mm_packs_epi32(mm5, mm7);
+
+        mm1 = _mm_packs_epi16(mm1, mm5);
+
+        uint32_t mask = _mm_movemask_epi8(mm1);
+        if (mask)
+        {
+            pos += atomics::detail::count_trailing_zeros(mask);
+            goto done;
+        }
+    }
+
+    if ((n - pos) >= 8u)
+    {
+        __m128i mm1 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos));
+        __m128i mm2 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 2u));
+        __m128i mm3 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 4u));
+        __m128i mm4 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 6u));
+
+        mm1 = _mm_cmpeq_epi64(mm1, mm_addr);
+        mm2 = _mm_cmpeq_epi64(mm2, mm_addr);
+        mm3 = _mm_cmpeq_epi64(mm3, mm_addr);
+        mm4 = _mm_cmpeq_epi64(mm4, mm_addr);
+
+        mm1 = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm1), _mm_castsi128_ps(mm2), _MM_SHUFFLE(2, 0, 2, 0)));
+        mm3 = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm3), _mm_castsi128_ps(mm4), _MM_SHUFFLE(2, 0, 2, 0)));
+
+        mm1 = _mm_packs_epi32(mm1, mm3);
+
+        uint32_t mask = _mm_movemask_epi8(mm1);
+        if (mask)
+        {
+            pos += atomics::detail::count_trailing_zeros(mask) / 2u;
+            goto done;
+        }
+
+        pos += 8u;
+    }
+
+    if ((n - pos) >= 4u)
+    {
+        __m128i mm1 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos));
+        __m128i mm2 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos + 2u));
+
+        mm1 = _mm_cmpeq_epi64(mm1, mm_addr);
+        mm2 = _mm_cmpeq_epi64(mm2, mm_addr);
+
+        mm1 = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(mm1), _mm_castsi128_ps(mm2), _MM_SHUFFLE(2, 0, 2, 0)));
+
+        uint32_t mask = _mm_movemask_ps(_mm_castsi128_ps(mm1));
+        if (mask)
+        {
+            pos += atomics::detail::count_trailing_zeros(mask);
+            goto done;
+        }
+
+        pos += 4u;
+    }
+
+    if (pos < n)
+    {
+        __m128i mm1 = _mm_load_si128(reinterpret_cast< const __m128i* >(addrs + pos));
+
+        mm1 = _mm_cmpeq_epi64(mm1, mm_addr);
+        uint32_t mask = _mm_movemask_pd(_mm_castsi128_pd(mm1));
+        if (mask)
+        {
+            pos += atomics::detail::count_trailing_zeros(mask);
+            goto done;
+        }
+
+        pos += 2u;
+    }
+
+done:
+    return pos;
+}
+
+} // namespace detail
+} // namespace atomics
+} // namespace boost
+
+#include <boost/atomic/detail/footer.hpp>
+
+#endif // BOOST_ARCH_X86 && defined(BOOST_ATOMIC_DETAIL_SIZEOF_POINTER) && (BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8)

externals/boost/libs/atomic/src/lock_pool_init1.ipp

@@ -0,0 +1,13 @@
+/*
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * Copyright (c) 2020 Andrey Semashev
+ */
+
+#if BOOST_PP_ITERATION() > 1
+,
+#endif
+
+{ BOOST_ATOMIC_LOCK_STATE_INIT }

externals/boost/libs/atomic/src/lock_pool_init256.ipp

@@ -0,0 +1,79 @@
+/*
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * Copyright (c) 2020 Andrey Semashev
+ */
+
+#if BOOST_PP_ITERATION() > 1
+,
+#endif
+
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT },
+{ BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }, { BOOST_ATOMIC_LOCK_STATE_INIT }

externals/boost/libs/atomic/src/wait_on_address.cpp

@@ -0,0 +1,113 @@
+/*
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * Copyright (c) 2020 Andrey Semashev
+ */
+/*!
+ * \file   wait_on_address.cpp
+ *
+ * This file contains implementation of runtime detection of \c WaitOnAddress and related APIs on Windows.
+ *
+ * https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-waitonaddress
+ * https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-wakebyaddresssingle
+ * https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-wakebyaddressall
+ */
+
+// Include boost/winapi/config.hpp first to make sure target Windows version is selected by Boost.WinAPI
+#include <boost/winapi/config.hpp>
+
+#include <boost/winapi/basic_types.hpp>
+
+#include <boost/atomic/detail/config.hpp>
+#include <boost/atomic/detail/link.hpp>
+#include <boost/atomic/detail/once_flag.hpp>
+#include <boost/atomic/detail/wait_on_address.hpp>
+
+#if BOOST_WINAPI_PARTITION_DESKTOP || BOOST_WINAPI_PARTITION_SYSTEM
+
+#include <boost/static_assert.hpp>
+#include <boost/memory_order.hpp>
+#include <boost/winapi/thread.hpp>
+#include <boost/winapi/get_proc_address.hpp>
+#include <boost/winapi/dll.hpp>
+
+#include <boost/atomic/detail/core_operations.hpp>
+
+#endif // BOOST_WINAPI_PARTITION_DESKTOP || BOOST_WINAPI_PARTITION_SYSTEM
+
+#include <boost/atomic/detail/header.hpp>
+
+namespace boost {
+namespace atomics {
+namespace detail {
+
+BOOST_ATOMIC_DECL wait_on_address_t* wait_on_address = NULL;
+BOOST_ATOMIC_DECL wake_by_address_t* wake_by_address_single = NULL;
+BOOST_ATOMIC_DECL wake_by_address_t* wake_by_address_all = NULL;
+
+#if BOOST_WINAPI_PARTITION_DESKTOP || BOOST_WINAPI_PARTITION_SYSTEM
+
+BOOST_ATOMIC_DECL once_flag wait_functions_once_flag = { 2u };
+
+BOOST_ATOMIC_DECL void initialize_wait_functions() BOOST_NOEXCEPT
+{
+    BOOST_STATIC_ASSERT_MSG(once_flag_operations::is_always_lock_free, "Boost.Atomic unsupported target platform: native atomic operations not implemented for bytes");
+
+    once_flag_operations::storage_type old_val = once_flag_operations::load(wait_functions_once_flag.m_flag, boost::memory_order_acquire);
+    while (true)
+    {
+        if (old_val == 2u)
+        {
+            if (BOOST_UNLIKELY(!once_flag_operations::compare_exchange_strong(wait_functions_once_flag.m_flag, old_val, 1u, boost::memory_order_relaxed, boost::memory_order_relaxed)))
+                continue;
+
+            boost::winapi::HMODULE_ kernel_base = boost::winapi::get_module_handle(L"api-ms-win-core-synch-l1-2-0.dll");
+            if (BOOST_LIKELY(kernel_base != NULL))
+            {
+                wait_on_address_t* woa = (wait_on_address_t*)boost::winapi::get_proc_address(kernel_base, "WaitOnAddress");
+                if (BOOST_LIKELY(woa != NULL))
+                {
+                    wake_by_address_t* wbas = (wake_by_address_t*)boost::winapi::get_proc_address(kernel_base, "WakeByAddressSingle");
+                    wake_by_address_t* wbaa = (wake_by_address_t*)boost::winapi::get_proc_address(kernel_base, "WakeByAddressAll");
+
+                    if (BOOST_LIKELY(wbas != NULL && wbaa != NULL))
+                    {
+                        wait_on_address = woa;
+                        wake_by_address_single = wbas;
+                        wake_by_address_all = wbaa;
+                    }
+                }
+            }
+
+            once_flag_operations::store(wait_functions_once_flag.m_flag, 0u, boost::memory_order_release);
+            break;
+        }
+        else if (old_val == 1u)
+        {
+            boost::winapi::SwitchToThread();
+            old_val = once_flag_operations::load(wait_functions_once_flag.m_flag, boost::memory_order_acquire);
+        }
+        else
+        {
+            break;
+        }
+    }
+}
+
+#else // BOOST_WINAPI_PARTITION_DESKTOP || BOOST_WINAPI_PARTITION_SYSTEM
+
+BOOST_ATOMIC_DECL once_flag wait_functions_once_flag = { 0u };
+
+BOOST_ATOMIC_DECL void initialize_wait_functions() BOOST_NOEXCEPT
+{
+}
+
+#endif // BOOST_WINAPI_PARTITION_DESKTOP || BOOST_WINAPI_PARTITION_SYSTEM
+
+} // namespace detail
+} // namespace atomics
+} // namespace boost
+
+#include <boost/atomic/detail/footer.hpp>

externals/boost/libs/atomic/src/x86_vector_tools.hpp

@@ -0,0 +1,52 @@
+/*
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * Copyright (c) 2020 Andrey Semashev
+ */
+/*!
+ * \file   x86_vector_tools.hpp
+ *
+ * This file contains common tools for x86 vectorization
+ */
+
+#ifndef BOOST_ATOMIC_X86_VECTOR_TOOLS_HPP_INCLUDED_
+#define BOOST_ATOMIC_X86_VECTOR_TOOLS_HPP_INCLUDED_
+
+#include <boost/predef/architecture/x86.h>
+#include <boost/atomic/detail/int_sizes.hpp>
+
+#if BOOST_ARCH_X86 && defined(BOOST_ATOMIC_DETAIL_SIZEOF_POINTER) && (BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8)
+
+#include <emmintrin.h>
+#include <boost/cstdint.hpp>
+#include <boost/atomic/detail/intptr.hpp>
+#include <boost/atomic/detail/config.hpp>
+
+#include <boost/atomic/detail/header.hpp>
+
+namespace boost {
+namespace atomics {
+namespace detail {
+
+BOOST_FORCEINLINE __m128i mm_set1_epiptr(uintptr_t ptr)
+{
+#if !defined(_MSC_VER) || _MSC_FULL_VER >= 190024210
+    return _mm_set1_epi64x(ptr);
+#else
+    // MSVC up until 14.0 update 3 doesn't provide _mm_set1_epi64x
+    uint32_t lo = static_cast< uint32_t >(ptr), hi = static_cast< uint32_t >(ptr >> 32);
+    return _mm_set_epi32(hi, lo, hi, lo);
+#endif
+}
+
+} // namespace detail
+} // namespace atomics
+} // namespace boost
+
+#include <boost/atomic/detail/footer.hpp>
+
+#endif // BOOST_ARCH_X86 && defined(BOOST_ATOMIC_DETAIL_SIZEOF_POINTER) && (BOOST_ATOMIC_DETAIL_SIZEOF_POINTER == 8)
+
+#endif // BOOST_ATOMIC_X86_VECTOR_TOOLS_HPP_INCLUDED_